diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml index 5f6a37f177a..b60a72a9b13 100644 --- a/.evergreen/config.in.yml +++ b/.evergreen/config.in.yml @@ -130,123 +130,40 @@ functions: DRIVERS_TOOLS: ${DRIVERS_TOOLS} "run tests": - - command: shell.exec - type: test + - command: ec2.assume_role params: - silent: true - working_dir: "src" - script: | - if [ -n "${CLIENT_ENCRYPTION}" ]; then - cat < prepare_client_encryption.sh - export CLIENT_ENCRYPTION=${CLIENT_ENCRYPTION} - export RUN_WITH_MONGOCRYPTD=${RUN_WITH_MONGOCRYPTD} - export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}' - export FLE_AWS_KEY2='${FLE_AWS_KEY2}' - export FLE_AWS_SECRET2='${FLE_AWS_SECRET2}' - export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}' - export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}' - export AWS_REGION='${AWS_REGION}' - export AWS_CMK_ID='${AWS_CMK_ID}' - export AWS_DEFAULT_REGION='us-east-1' - export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem" - export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem" - EOT - fi - - command: shell.exec + role_arn: ${OIDC_AWS_ROLE_ARN} + - command: subprocess.exec type: test params: + env: + TEST_CSFLE: "true" + add_expansions_to_env: true working_dir: "src" timeout_secs: 300 - shell: bash - script: | - ${PREPARE_SHELL} - - if [ -n "${CLIENT_ENCRYPTION}" ]; then - # Disable xtrace (just in case it was accidentally set). - set +x - . ./prepare_client_encryption.sh - rm -f ./prepare_client_encryption.sh - fi - - export VERSION=${VERSION} - export DRIVERS_TOOLS=${DRIVERS_TOOLS} - - if [ -z "${RUN_WITH_MONGOCRYPTD}" ]; then - # Set up crypt shared lib if we don't want to use mongocryptd - source ${PROJECT_DIRECTORY}/.evergreen/prepare-crypt-shared-lib.sh - echo "CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" - else - echo "CRYPT_SHARED_LIB_PATH not set; using mongocryptd" - fi - - TEST_NPM_SCRIPT="${TEST_NPM_SCRIPT|check:integration-coverage}" \ - MONGODB_URI="${MONGODB_URI}" \ - AUTH=${AUTH} SSL=${SSL} TEST_CSFLE=true \ - MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ - SKIP_DEPS=${SKIP_DEPS|1} \ - bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + binary: bash + args: + - .evergreen/run-tests.sh "run serverless tests": - command: timeout.update params: exec_timeout_secs: 1800 - - command: shell.exec - type: test + - command: ec2.assume_role params: - silent: true - working_dir: src - script: | - cat < prepare_client_encryption.sh - export CLIENT_ENCRYPTION=${CLIENT_ENCRYPTION} - export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}' - export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}' - export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}' - export FLE_AWS_KEY2='${FLE_AWS_KEY2}' - export FLE_AWS_SECRET2='${FLE_AWS_SECRET2}' - export AWS_REGION='${AWS_REGION}' - export AWS_CMK_ID='${AWS_CMK_ID}' - export AWS_DEFAULT_REGION='us-east-1' - export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem" - export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem" - EOT - - command: shell.exec + role_arn: ${OIDC_AWS_ROLE_ARN} + - command: subprocess.exec type: test params: - working_dir: src timeout_secs: 300 - shell: bash - script: | - ${PREPARE_SHELL} - - # Disable xtrace (just in case it was accidentally set). - set +x - source ./prepare_client_encryption.sh - rm -f ./prepare_client_encryption.sh - - export VERSION=${VERSION} - export DRIVERS_TOOLS=${DRIVERS_TOOLS} - - source ${PROJECT_DIRECTORY}/.evergreen/prepare-crypt-shared-lib.sh - - echo "CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" - - # Get access to the AWS temporary credentials: - echo "adding temporary AWS credentials to environment" - # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN - pushd "$DRIVERS_TOOLS"/.evergreen/csfle - . ./activate-kmstlsvenv.sh - . ./set-temp-creds.sh - popd - - export MONGODB_API_VERSION="${MONGODB_API_VERSION}" - export AUTH="auth" - export SSL="ssl" - export TEST_CSFLE=true - - source secrets-export.sh - source serverless.env - - bash ${PROJECT_DIRECTORY}/.evergreen/run-serverless-tests.sh + working_dir: src + binary: bash + env: + AUTH: 'auth' + SSL: 'ssl' + add_expansions_to_env: true + args: + - .evergreen/run-serverless-tests.sh "start-load-balancer": - command: shell.exec @@ -264,43 +181,26 @@ functions: bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop "run-lb-tests": - - command: shell.exec + - command: subprocess.exec type: test params: - shell: bash + add_expansions_to_env: true + binary: bash working_dir: src timeout_secs: 300 - script: | - ${PREPARE_SHELL} - - MONGODB_URI="${MONGODB_URI}" \ - AUTH=${AUTH} \ - SSL=${SSL} \ - MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ - SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" \ - MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" \ - TOPOLOGY="${TOPOLOGY}" \ - SKIP_DEPS=${SKIP_DEPS|1} \ - LOAD_BALANCER="${LOAD_BALANCER}" \ - bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + args: + - .evergreen/run-tests.sh "run-compression-tests": - - command: shell.exec + - command: subprocess.exec type: test params: + binary: bash working_dir: src timeout_secs: 300 - script: | - ${PREPARE_SHELL} - - MONGODB_URI="${MONGODB_URI}" \ - AUTH=${AUTH} \ - SSL=${SSL} \ - MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ - TOPOLOGY="${TOPOLOGY}" \ - COMPRESSOR="${COMPRESSOR}" \ - SKIP_DEPS=${SKIP_DEPS|1} \ - bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + add_expansions_to_env: true + args: + - .evergreen/run-tests.sh "install package": - command: shell.exec @@ -492,44 +392,17 @@ functions: - .evergreen/run-atlas-tests.sh "run socks5 tests": - - command: shell.exec - type: test + - command: ec2.assume_role params: - silent: true - working_dir: "src" - script: | - ${PREPARE_SHELL} - cat < prepare_client_encryption.sh - export CLIENT_ENCRYPTION='${CLIENT_ENCRYPTION}' - export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}' - export FLE_AWS_KEY2='${FLE_AWS_KEY2}' - export FLE_AWS_SECRET2='${FLE_AWS_SECRET2}' - export AWS_REGION='${AWS_REGION}' - export AWS_CMK_ID='${AWS_CMK_ID}' - export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}' - export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}' - EOT - - command: shell.exec + role_arn: ${OIDC_AWS_ROLE_ARN} + - command: subprocess.exec type: test params: - working_dir: "src" - script: | - ${PREPARE_SHELL} - - export PYTHON_BINARY=$([ "Windows_NT" = "$OS" ] && echo "/cygdrive/c/python/python38/python.exe" || echo "/opt/mongodbtoolchain/v3/bin/python3") - export PROJECT_DIRECTORY="$(pwd)" - export DRIVERS_TOOLS="${DRIVERS_TOOLS}" - export NODE_LTS_VERSION='${NODE_LTS_VERSION}' - export MONGODB_URI="${MONGODB_URI}" - export TEST_SOCKS5_CSFLE="${TEST_SOCKS5_CSFLE}" - export SSL="${SSL}" - - # Disable xtrace (just in case it was accidentally set). - set +x - . ./prepare_client_encryption.sh - rm -f ./prepare_client_encryption.sh - - bash ${PROJECT_DIRECTORY}/.evergreen/run-socks5-tests.sh + add_expansions_to_env: true + working_dir: src + binary: bash + args: + - .evergreen/run-socks5-tests.sh "run kerberos tests": - command: subprocess.exec @@ -939,46 +812,17 @@ functions: --fault revoked "run custom csfle tests": - - command: shell.exec - type: test + - command: ec2.assume_role params: - silent: true - working_dir: "src" - script: | - ${PREPARE_SHELL} - cat < prepare_client_encryption.sh - export CLIENT_ENCRYPTION='${CLIENT_ENCRYPTION}' - export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}' - export FLE_AWS_KEY2='${FLE_AWS_KEY2}' - export FLE_AWS_SECRET2='${FLE_AWS_SECRET2}' - export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}' - export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}' - export AWS_REGION='${AWS_REGION}' - export AWS_CMK_ID='${AWS_CMK_ID}' - EOT - - command: shell.exec + role_arn: ${OIDC_AWS_ROLE_ARN} + - command: subprocess.exec type: test params: - working_dir: "src" - timeout_secs: 60 - shell: bash - script: | - ${PREPARE_SHELL} - - # Disable xtrace (just in case it was accidentally set). - set +x - source ./prepare_client_encryption.sh - rm -f ./prepare_client_encryption.sh - - export VERSION=${VERSION} - export DRIVERS_TOOLS=${DRIVERS_TOOLS} - - source ${PROJECT_DIRECTORY}/.evergreen/prepare-crypt-shared-lib.sh - export MONGODB_URI="${MONGODB_URI}" - - echo "CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" - - bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh + working_dir: src + add_expansions_to_env: true + binary: bash + args: + - .evergreen/run-custom-csfle-tests.sh "run lambda handler example tests": - command: subprocess.exec diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 726592cd346..ac7b533a873 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -102,122 +102,39 @@ functions: env: DRIVERS_TOOLS: ${DRIVERS_TOOLS} run tests: - - command: shell.exec - type: test + - command: ec2.assume_role params: - silent: true - working_dir: src - script: | - if [ -n "${CLIENT_ENCRYPTION}" ]; then - cat < prepare_client_encryption.sh - export CLIENT_ENCRYPTION=${CLIENT_ENCRYPTION} - export RUN_WITH_MONGOCRYPTD=${RUN_WITH_MONGOCRYPTD} - export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}' - export FLE_AWS_KEY2='${FLE_AWS_KEY2}' - export FLE_AWS_SECRET2='${FLE_AWS_SECRET2}' - export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}' - export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}' - export AWS_REGION='${AWS_REGION}' - export AWS_CMK_ID='${AWS_CMK_ID}' - export AWS_DEFAULT_REGION='us-east-1' - export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem" - export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem" - EOT - fi - - command: shell.exec + role_arn: ${OIDC_AWS_ROLE_ARN} + - command: subprocess.exec type: test params: + env: + TEST_CSFLE: 'true' + add_expansions_to_env: true working_dir: src timeout_secs: 300 - shell: bash - script: | - ${PREPARE_SHELL} - - if [ -n "${CLIENT_ENCRYPTION}" ]; then - # Disable xtrace (just in case it was accidentally set). - set +x - . ./prepare_client_encryption.sh - rm -f ./prepare_client_encryption.sh - fi - - export VERSION=${VERSION} - export DRIVERS_TOOLS=${DRIVERS_TOOLS} - - if [ -z "${RUN_WITH_MONGOCRYPTD}" ]; then - # Set up crypt shared lib if we don't want to use mongocryptd - source ${PROJECT_DIRECTORY}/.evergreen/prepare-crypt-shared-lib.sh - echo "CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" - else - echo "CRYPT_SHARED_LIB_PATH not set; using mongocryptd" - fi - - TEST_NPM_SCRIPT="${TEST_NPM_SCRIPT|check:integration-coverage}" \ - MONGODB_URI="${MONGODB_URI}" \ - AUTH=${AUTH} SSL=${SSL} TEST_CSFLE=true \ - MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ - SKIP_DEPS=${SKIP_DEPS|1} \ - bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + binary: bash + args: + - .evergreen/run-tests.sh run serverless tests: - command: timeout.update params: exec_timeout_secs: 1800 - - command: shell.exec - type: test + - command: ec2.assume_role params: - silent: true - working_dir: src - script: | - cat < prepare_client_encryption.sh - export CLIENT_ENCRYPTION=${CLIENT_ENCRYPTION} - export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}' - export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}' - export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}' - export FLE_AWS_KEY2='${FLE_AWS_KEY2}' - export FLE_AWS_SECRET2='${FLE_AWS_SECRET2}' - export AWS_REGION='${AWS_REGION}' - export AWS_CMK_ID='${AWS_CMK_ID}' - export AWS_DEFAULT_REGION='us-east-1' - export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem" - export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem" - EOT - - command: shell.exec + role_arn: ${OIDC_AWS_ROLE_ARN} + - command: subprocess.exec type: test params: - working_dir: src timeout_secs: 300 - shell: bash - script: | - ${PREPARE_SHELL} - - # Disable xtrace (just in case it was accidentally set). - set +x - source ./prepare_client_encryption.sh - rm -f ./prepare_client_encryption.sh - - export VERSION=${VERSION} - export DRIVERS_TOOLS=${DRIVERS_TOOLS} - - source ${PROJECT_DIRECTORY}/.evergreen/prepare-crypt-shared-lib.sh - - echo "CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" - - # Get access to the AWS temporary credentials: - echo "adding temporary AWS credentials to environment" - # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN - pushd "$DRIVERS_TOOLS"/.evergreen/csfle - . ./activate-kmstlsvenv.sh - . ./set-temp-creds.sh - popd - - export MONGODB_API_VERSION="${MONGODB_API_VERSION}" - export AUTH="auth" - export SSL="ssl" - export TEST_CSFLE=true - - source secrets-export.sh - source serverless.env - - bash ${PROJECT_DIRECTORY}/.evergreen/run-serverless-tests.sh + working_dir: src + binary: bash + env: + AUTH: auth + SSL: ssl + add_expansions_to_env: true + args: + - .evergreen/run-serverless-tests.sh start-load-balancer: - command: shell.exec params: @@ -232,42 +149,25 @@ functions: script: | bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop run-lb-tests: - - command: shell.exec + - command: subprocess.exec type: test params: - shell: bash + add_expansions_to_env: true + binary: bash working_dir: src timeout_secs: 300 - script: | - ${PREPARE_SHELL} - - MONGODB_URI="${MONGODB_URI}" \ - AUTH=${AUTH} \ - SSL=${SSL} \ - MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ - SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" \ - MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" \ - TOPOLOGY="${TOPOLOGY}" \ - SKIP_DEPS=${SKIP_DEPS|1} \ - LOAD_BALANCER="${LOAD_BALANCER}" \ - bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + args: + - .evergreen/run-tests.sh run-compression-tests: - - command: shell.exec + - command: subprocess.exec type: test params: + binary: bash working_dir: src timeout_secs: 300 - script: | - ${PREPARE_SHELL} - - MONGODB_URI="${MONGODB_URI}" \ - AUTH=${AUTH} \ - SSL=${SSL} \ - MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ - TOPOLOGY="${TOPOLOGY}" \ - COMPRESSOR="${COMPRESSOR}" \ - SKIP_DEPS=${SKIP_DEPS|1} \ - bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + add_expansions_to_env: true + args: + - .evergreen/run-tests.sh install package: - command: shell.exec params: @@ -442,57 +342,17 @@ functions: args: - .evergreen/run-atlas-tests.sh run socks5 tests: - - command: shell.exec - type: test + - command: ec2.assume_role params: - silent: true - working_dir: src - script: | - ${PREPARE_SHELL} - cat < prepare_client_encryption.sh - export CLIENT_ENCRYPTION='${CLIENT_ENCRYPTION}' - export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}' - export FLE_AWS_KEY2='${FLE_AWS_KEY2}' - export FLE_AWS_SECRET2='${FLE_AWS_SECRET2}' - export AWS_REGION='${AWS_REGION}' - export AWS_CMK_ID='${AWS_CMK_ID}' - export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}' - export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}' - EOT - - command: shell.exec + role_arn: ${OIDC_AWS_ROLE_ARN} + - command: subprocess.exec type: test params: + add_expansions_to_env: true working_dir: src - script: > - ${PREPARE_SHELL} - - - export PYTHON_BINARY=$([ "Windows_NT" = "$OS" ] && echo "/cygdrive/c/python/python38/python.exe" || echo - "/opt/mongodbtoolchain/v3/bin/python3") - - export PROJECT_DIRECTORY="$(pwd)" - - export DRIVERS_TOOLS="${DRIVERS_TOOLS}" - - export NODE_LTS_VERSION='${NODE_LTS_VERSION}' - - export MONGODB_URI="${MONGODB_URI}" - - export TEST_SOCKS5_CSFLE="${TEST_SOCKS5_CSFLE}" - - export SSL="${SSL}" - - - # Disable xtrace (just in case it was accidentally set). - - set +x - - . ./prepare_client_encryption.sh - - rm -f ./prepare_client_encryption.sh - - - bash ${PROJECT_DIRECTORY}/.evergreen/run-socks5-tests.sh + binary: bash + args: + - .evergreen/run-socks5-tests.sh run kerberos tests: - command: subprocess.exec type: test @@ -910,46 +770,17 @@ functions: -v \ --fault revoked run custom csfle tests: - - command: shell.exec - type: test + - command: ec2.assume_role params: - silent: true - working_dir: src - script: | - ${PREPARE_SHELL} - cat < prepare_client_encryption.sh - export CLIENT_ENCRYPTION='${CLIENT_ENCRYPTION}' - export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}' - export FLE_AWS_KEY2='${FLE_AWS_KEY2}' - export FLE_AWS_SECRET2='${FLE_AWS_SECRET2}' - export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}' - export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}' - export AWS_REGION='${AWS_REGION}' - export AWS_CMK_ID='${AWS_CMK_ID}' - EOT - - command: shell.exec + role_arn: ${OIDC_AWS_ROLE_ARN} + - command: subprocess.exec type: test params: working_dir: src - timeout_secs: 60 - shell: bash - script: | - ${PREPARE_SHELL} - - # Disable xtrace (just in case it was accidentally set). - set +x - source ./prepare_client_encryption.sh - rm -f ./prepare_client_encryption.sh - - export VERSION=${VERSION} - export DRIVERS_TOOLS=${DRIVERS_TOOLS} - - source ${PROJECT_DIRECTORY}/.evergreen/prepare-crypt-shared-lib.sh - export MONGODB_URI="${MONGODB_URI}" - - echo "CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" - - bash ${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh + add_expansions_to_env: true + binary: bash + args: + - .evergreen/run-custom-csfle-tests.sh run lambda handler example tests: - command: subprocess.exec params: @@ -1790,6 +1621,8 @@ tasks: - {key: REQUIRE_API_VERSION, value: '1'} - {key: MONGODB_API_VERSION, value: '1'} - {key: AUTH, value: auth} + - {key: TEST_CSFLE, value: 'true'} + - {key: CLIENT_ENCRYPTION, value: 'true'} - func: install dependencies - func: bootstrap mongo-orchestration - func: bootstrap kms servers @@ -1831,6 +1664,8 @@ tasks: - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: auth} - {key: LOAD_BALANCER, value: 'true'} + - {key: CLIENT_ENCRYPTION, value: 'false'} + - {key: TEST_CSFLE, value: 'false'} - func: install dependencies - func: bootstrap mongo-orchestration - func: start-load-balancer @@ -1850,6 +1685,8 @@ tasks: - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: auth} - {key: LOAD_BALANCER, value: 'true'} + - {key: CLIENT_ENCRYPTION, value: 'false'} + - {key: TEST_CSFLE, value: 'false'} - func: install dependencies - func: bootstrap mongo-orchestration - func: start-load-balancer @@ -1869,6 +1706,8 @@ tasks: - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: auth} - {key: LOAD_BALANCER, value: 'true'} + - {key: CLIENT_ENCRYPTION, value: 'false'} + - {key: TEST_CSFLE, value: 'false'} - func: install dependencies - func: bootstrap mongo-orchestration - func: start-load-balancer @@ -1888,6 +1727,8 @@ tasks: - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: auth} - {key: LOAD_BALANCER, value: 'true'} + - {key: CLIENT_ENCRYPTION, value: 'false'} + - {key: TEST_CSFLE, value: 'false'} - func: install dependencies - func: bootstrap mongo-orchestration - func: start-load-balancer @@ -1907,6 +1748,8 @@ tasks: - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: auth} - {key: LOAD_BALANCER, value: 'true'} + - {key: CLIENT_ENCRYPTION, value: 'false'} + - {key: TEST_CSFLE, value: 'false'} - func: install dependencies - func: bootstrap mongo-orchestration - func: start-load-balancer @@ -1926,6 +1769,8 @@ tasks: - {key: TOPOLOGY, value: sharded_cluster} - {key: AUTH, value: auth} - {key: LOAD_BALANCER, value: 'true'} + - {key: CLIENT_ENCRYPTION, value: 'false'} + - {key: TEST_CSFLE, value: 'false'} - func: install dependencies - func: bootstrap mongo-orchestration - func: start-load-balancer @@ -1999,6 +1844,8 @@ tasks: - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: auth} - {key: COMPRESSOR, value: snappy} + - {key: CLIENT_ENCRYPTION, value: 'false'} + - {key: TEST_CSFLE, value: 'false'} - func: install dependencies - func: bootstrap mongo-orchestration - func: run-compression-tests @@ -2015,6 +1862,8 @@ tasks: - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: auth} - {key: COMPRESSOR, value: zstd} + - {key: CLIENT_ENCRYPTION, value: 'false'} + - {key: TEST_CSFLE, value: 'false'} - func: install dependencies - func: bootstrap mongo-orchestration - func: install package @@ -2034,6 +1883,8 @@ tasks: - {key: TOPOLOGY, value: replica_set} - {key: AUTH, value: auth} - {key: COMPRESSOR, value: zstd} + - {key: CLIENT_ENCRYPTION, value: 'false'} + - {key: TEST_CSFLE, value: 'false'} - func: install dependencies - func: bootstrap mongo-orchestration - func: run-compression-tests @@ -3733,6 +3584,7 @@ tasks: - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: '5.0'} - {key: TOPOLOGY, value: replica_set} + - {key: CLIENT_ENCRYPTION, value: 'true'} - func: install dependencies - func: bootstrap mongo-orchestration - func: bootstrap kms servers @@ -3750,6 +3602,7 @@ tasks: - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: rapid} - {key: TOPOLOGY, value: replica_set} + - {key: CLIENT_ENCRYPTION, value: 'true'} - func: install dependencies - func: bootstrap mongo-orchestration - func: bootstrap kms servers @@ -3767,6 +3620,7 @@ tasks: - {key: NPM_VERSION, value: '9'} - {key: VERSION, value: latest} - {key: TOPOLOGY, value: replica_set} + - {key: CLIENT_ENCRYPTION, value: 'true'} - func: install dependencies - func: bootstrap mongo-orchestration - func: bootstrap kms servers @@ -4690,7 +4544,8 @@ buildvariants: expansions: NODE_LTS_VERSION: 16 NPM_VERSION: 9 - CLIENT_ENCRYPTION: true + CLIENT_ENCRYPTION: 'true' + TEST_CSFLE: 'true' tasks: - test-latest-server - test-latest-replica_set @@ -4749,7 +4604,8 @@ buildvariants: expansions: NODE_LTS_VERSION: 18 NPM_VERSION: 10 - CLIENT_ENCRYPTION: true + CLIENT_ENCRYPTION: 'true' + TEST_CSFLE: 'true' tasks: - test-latest-server - test-latest-replica_set @@ -4808,7 +4664,8 @@ buildvariants: expansions: NODE_LTS_VERSION: 20 NPM_VERSION: latest - CLIENT_ENCRYPTION: true + CLIENT_ENCRYPTION: 'true' + TEST_CSFLE: 'true' tasks: - test-latest-server - test-latest-replica_set @@ -4867,7 +4724,8 @@ buildvariants: expansions: NODE_LTS_VERSION: 22 NPM_VERSION: latest - CLIENT_ENCRYPTION: true + CLIENT_ENCRYPTION: 'true' + TEST_CSFLE: 'true' tasks: - test-latest-server - test-latest-replica_set @@ -4984,6 +4842,8 @@ buildvariants: expansions: NODE_LTS_VERSION: 16 NPM_VERSION: 9 + CLIENT_ENCRYPTION: 'false' + TEST_CSFLE: 'false' tasks: - test-latest-server - test-latest-replica_set @@ -5031,6 +4891,8 @@ buildvariants: expansions: NODE_LTS_VERSION: 18 NPM_VERSION: 10 + CLIENT_ENCRYPTION: 'false' + TEST_CSFLE: 'false' tasks: - test-latest-server - test-latest-replica_set @@ -5078,6 +4940,8 @@ buildvariants: expansions: NODE_LTS_VERSION: 20 NPM_VERSION: latest + CLIENT_ENCRYPTION: 'false' + TEST_CSFLE: 'false' tasks: - test-latest-server - test-latest-replica_set @@ -5125,6 +4989,8 @@ buildvariants: expansions: NODE_LTS_VERSION: 22 NPM_VERSION: latest + CLIENT_ENCRYPTION: 'false' + TEST_CSFLE: 'false' tasks: - test-latest-server - test-latest-replica_set diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js index 450f3513ff8..b263dc053cb 100644 --- a/.evergreen/generate_evergreen_tasks.js +++ b/.evergreen/generate_evergreen_tasks.js @@ -105,7 +105,9 @@ BASE_TASKS.push({ TOPOLOGY: 'server', REQUIRE_API_VERSION: '1', MONGODB_API_VERSION: '1', - AUTH: 'auth' + AUTH: 'auth', + TEST_CSFLE: 'true', + CLIENT_ENCRYPTION: 'true' }), { func: 'install dependencies' }, { func: 'bootstrap mongo-orchestration' }, @@ -146,7 +148,9 @@ TASKS.push( VERSION: ver, TOPOLOGY: 'sharded_cluster', AUTH: 'auth', - LOAD_BALANCER: 'true' + LOAD_BALANCER: 'true', + CLIENT_ENCRYPTION: 'false', + TEST_CSFLE: 'false' }), { func: 'install dependencies' }, { func: 'bootstrap mongo-orchestration' }, @@ -208,7 +212,7 @@ TASKS.push( { func: 'run socks5 tests' } ] } - ] + ] ); TASKS.push({ @@ -219,7 +223,9 @@ TASKS.push({ VERSION: 'latest', TOPOLOGY: 'replica_set', AUTH: 'auth', - COMPRESSOR: 'snappy' + COMPRESSOR: 'snappy', + CLIENT_ENCRYPTION: 'false', + TEST_CSFLE: 'false' }), { func: 'install dependencies' }, { func: 'bootstrap mongo-orchestration' }, @@ -235,7 +241,9 @@ TASKS.push({ VERSION: 'latest', TOPOLOGY: 'replica_set', AUTH: 'auth', - COMPRESSOR: 'zstd' + COMPRESSOR: 'zstd', + CLIENT_ENCRYPTION: 'false', + TEST_CSFLE: 'false' }), { func: 'install dependencies' }, { func: 'bootstrap mongo-orchestration' }, @@ -257,7 +265,9 @@ TASKS.push({ VERSION: 'latest', TOPOLOGY: 'replica_set', AUTH: 'auth', - COMPRESSOR: 'zstd' + COMPRESSOR: 'zstd', + CLIENT_ENCRYPTION: 'false', + TEST_CSFLE: 'false' }), { func: 'install dependencies' }, { func: 'bootstrap mongo-orchestration' }, @@ -422,9 +432,8 @@ for (const { const expansions = { NODE_LTS_VERSION, NPM_VERSION }; const taskNames = tasks.map(({ name }) => name); - if (clientEncryption) { - expansions.CLIENT_ENCRYPTION = true; - } + expansions.CLIENT_ENCRYPTION = String(!!clientEncryption) + expansions.TEST_CSFLE = expansions.CLIENT_ENCRYPTION BUILD_VARIANTS.push({ name, display_name, run_on, expansions, tasks: taskNames }); } @@ -668,7 +677,8 @@ for (const version of ['5.0', 'rapid', 'latest']) { NODE_LTS_VERSION: LOWEST_LTS, NPM_VERSION: 9, VERSION: version, - TOPOLOGY: 'replica_set' + TOPOLOGY: 'replica_set', + CLIENT_ENCRYPTION: true }), { func: 'install dependencies' }, { func: 'bootstrap mongo-orchestration' }, diff --git a/.evergreen/prepare-crypt-shared-lib.sh b/.evergreen/prepare-crypt-shared-lib.sh index 8d8065880e7..b716f9e3401 100755 --- a/.evergreen/prepare-crypt-shared-lib.sh +++ b/.evergreen/prepare-crypt-shared-lib.sh @@ -30,5 +30,5 @@ else CRYPT_SHARED_LIB_PATH=$(cygpath -m $CRYPT_SHARED_LIB_PATH) fi echo "CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" - export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} + echo "export CRYPT_SHARED_LIB_PATH=$CRYPT_SHARED_LIB_PATH" >crypt_shared.sh fi diff --git a/.evergreen/run-azure-kms-tests.sh b/.evergreen/run-azure-kms-tests.sh index dfdcd032230..3168f7b0bb3 100644 --- a/.evergreen/run-azure-kms-tests.sh +++ b/.evergreen/run-azure-kms-tests.sh @@ -17,6 +17,5 @@ export MONGODB_URI="mongodb://localhost:27017" export EXPECTED_AZUREKMS_OUTCOME=${EXPECTED_AZUREKMS_OUTCOME:-omitted} export TEST_CSFLE=true -export CSFLE_KMS_PROVIDERS='not json' npx mocha --config test/mocha_mongodb.json test/integration/client-side-encryption/client_side_encryption.prose.19.on_demand_azure.test.ts diff --git a/.evergreen/run-custom-csfle-tests.sh b/.evergreen/run-custom-csfle-tests.sh index 787ef8bbba8..5465987dacf 100644 --- a/.evergreen/run-custom-csfle-tests.sh +++ b/.evergreen/run-custom-csfle-tests.sh @@ -1,33 +1,14 @@ #! /usr/bin/env bash set +o xtrace # Do not write AWS credentials to stderr -# Initiail checks for running these tests -if [ -z ${AWS_ACCESS_KEY_ID+omitted} ]; then echo "AWS_ACCESS_KEY_ID is unset" && exit 1; fi -if [ -z ${AWS_SECRET_ACCESS_KEY+omitted} ]; then echo "AWS_SECRET_ACCESS_KEY is unset" && exit 1; fi -if [ -z ${CSFLE_KMS_PROVIDERS+omitted} ]; then echo "CSFLE_KMS_PROVIDERS is unset" && exit 1; fi - -export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} -export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} -export CSFLE_KMS_PROVIDERS=${CSFLE_KMS_PROVIDERS} -export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} -echo "csfle CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" +source .evergreen/setup-fle.sh source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh -set -o xtrace # Write all commands first to stderr -set -o errexit # Exit the script with error if any of the commands fail - -# Get access to the AWS temporary credentials: -echo "adding temporary AWS credentials to environment" -# CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN -pushd "$DRIVERS_TOOLS"/.evergreen/csfle -. ./activate-kmstlsvenv.sh -. ./set-temp-creds.sh -popd +set -o xtrace # Write all commands first to stderr +set -o errexit # Exit the script with error if any of the commands fail export MONGODB_URI=${MONGODB_URI} -export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem" -export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem" export TEST_CSFLE=true npm run check:csfle diff --git a/.evergreen/run-gcp-kms-tests.sh b/.evergreen/run-gcp-kms-tests.sh index 4ce06b0301b..3f932681ea3 100644 --- a/.evergreen/run-gcp-kms-tests.sh +++ b/.evergreen/run-gcp-kms-tests.sh @@ -19,6 +19,5 @@ export MONGODB_URI="mongodb://localhost:27017" export EXPECTED_GCPKMS_OUTCOME=${EXPECTED_GCPKMS_OUTCOME:-omitted} export TEST_CSFLE=true -export CSFLE_KMS_PROVIDERS='not json' npx mocha --config test/mocha_mongodb.json test/integration/client-side-encryption/client_side_encryption.prose.17.on_demand_gcp.test.ts diff --git a/.evergreen/run-serverless-tests.sh b/.evergreen/run-serverless-tests.sh index 547d3a01866..b134fd4be66 100755 --- a/.evergreen/run-serverless-tests.sh +++ b/.evergreen/run-serverless-tests.sh @@ -1,5 +1,12 @@ #!/usr/bin/env bash +source secrets-export.sh +source serverless.env + +# next, set up FLE +export TEST_CSFLE=true +source .evergreen/setup-fle.sh + source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh if [ -z ${SERVERLESS+omitted} ]; then echo "SERVERLESS is unset" && exit 1; fi diff --git a/.evergreen/run-socks5-tests.sh b/.evergreen/run-socks5-tests.sh index e85566b7add..9f45645cbdc 100644 --- a/.evergreen/run-socks5-tests.sh +++ b/.evergreen/run-socks5-tests.sh @@ -2,29 +2,15 @@ source $DRIVERS_TOOLS/.evergreen/init-node-and-npm-env.sh -set -o errexit # Exit the script with error if any of the commands fail +set -o errexit # Exit the script with error if any of the commands fail set -o xtrace # For debuggability, no external credentials are used here function setup_fle() { - export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - export CSFLE_KMS_PROVIDERS=${CSFLE_KMS_PROVIDERS} - export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH} - echo "csfle CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" + source .evergreen/setup-fle.sh - set -o xtrace # Write all commands first to stderr - set -o errexit # Exit the script with error if any of the commands fail + set -o xtrace # Write all commands first to stderr + set -o errexit # Exit the script with error if any of the commands fail - # Get access to the AWS temporary credentials: - echo "adding temporary AWS credentials to environment" - # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN - pushd "$DRIVERS_TOOLS"/.evergreen/csfle - . ./activate-kmstlsvenv.sh - . ./set-temp-creds.sh - popd - - export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem" - export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem" export TEST_CSFLE=true } @@ -35,8 +21,8 @@ PYTHON_BINARY=$(bash -c ". $DRIVERS_TOOLS/.evergreen/find-python3.sh && ensure_p # ssl setup SSL=${SSL:-nossl} if [ "$SSL" != "nossl" ]; then - export SSL_KEY_FILE="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem" - export SSL_CA_FILE="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem" + export SSL_KEY_FILE="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem" + export SSL_CA_FILE="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem" fi # Grab a connection string that only refers to *one* of the hosts in MONGODB_URI @@ -51,14 +37,17 @@ if [ "Windows_NT" = "$OS" ]; then SOCKS5_SERVER_SCRIPT=$(cygpath -w "$SOCKS5_SERVER_SCRIPT") fi +if [[ $TEST_SOCKS5_CSFLE == "true" ]]; then + setup_fle +fi + # First, test with Socks5 + authentication required "$PYTHON_BINARY" "$SOCKS5_SERVER_SCRIPT" --port 1080 --auth username:p4ssw0rd --map "127.0.0.1:12345 to $FIRST_HOST" & SOCKS5_PROXY_PID=$! if [[ $TEST_SOCKS5_CSFLE == "true" ]]; then - setup_fle - [ "$SSL" == "nossl" ] && [[ "$OSTYPE" == "linux-gnu"* ]] && \ - env MONGODB_URI='mongodb://127.0.0.1:12345/?proxyHost=127.0.0.1&proxyUsername=username&proxyPassword=p4ssw0rd' \ - npm run check:csfle + [ "$SSL" == "nossl" ] && [[ "$OSTYPE" == "linux-gnu"* ]] && + env MONGODB_URI='mongodb://127.0.0.1:12345/?proxyHost=127.0.0.1&proxyUsername=username&proxyPassword=p4ssw0rd' \ + npm run check:csfle else env SOCKS5_CONFIG='["127.0.0.1",1080,"username","p4ssw0rd"]' npm run check:socks5 fi @@ -68,10 +57,9 @@ kill $SOCKS5_PROXY_PID "$PYTHON_BINARY" "$SOCKS5_SERVER_SCRIPT" --port 1081 --map "127.0.0.1:12345 to $FIRST_HOST" & SOCKS5_PROXY_PID=$! if [[ $TEST_SOCKS5_CSFLE == "true" ]]; then - setup_fle - [ "$SSL" == "nossl" ] && [[ "$OSTYPE" == "linux-gnu"* ]] && \ + [ "$SSL" == "nossl" ] && [[ "$OSTYPE" == "linux-gnu"* ]] && env MONGODB_URI='mongodb://127.0.0.1:12345/?proxyHost=127.0.0.1&proxyPort=1081' \ - npm run check:csfle + npm run check:csfle else env SOCKS5_CONFIG='["127.0.0.1",1081]' npm run check:socks5 fi diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 9bad420f7f3..68da6c2dd19 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -15,6 +15,15 @@ AUTH=${AUTH:-noauth} MONGODB_URI=${MONGODB_URI:-} TEST_NPM_SCRIPT=${TEST_NPM_SCRIPT:-check:integration-coverage} COMPRESSOR=${COMPRESSOR:-} +SKIP_DEPS=${SKIP_DEPS:-true} + +if [ "${CLIENT_ENCRYPTION}" == "true" ]; then + export RUN_WITH_MONGOCRYPTD + source .evergreen/setup-fle.sh +elif [ "${CLIENT_ENCRYPTION}" != "false" ]; then + echo "Invalid configuration for CLIENT_ENCRYPTION: ${CLIENT_ENCRYPTION}" + exit 1 +fi # ssl setup SSL=${SSL:-nossl} @@ -40,20 +49,6 @@ if [ "$COMPRESSOR" != "" ]; then fi fi -# only run FLE tets on hosts we explicitly choose to test on -if [[ -z "${CLIENT_ENCRYPTION}" ]]; then - unset AWS_ACCESS_KEY_ID; - unset AWS_SECRET_ACCESS_KEY; -else - pushd "$DRIVERS_TOOLS/.evergreen/csfle" - . ./activate-kmstlsvenv.sh - # Get access to the AWS temporary credentials: - echo "adding temporary AWS credentials to environment" - # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN - source set-temp-creds.sh - popd -fi - npm install @mongodb-js/zstd npm install snappy diff --git a/.evergreen/setup-fle.sh b/.evergreen/setup-fle.sh new file mode 100644 index 00000000000..82877177ee2 --- /dev/null +++ b/.evergreen/setup-fle.sh @@ -0,0 +1,13 @@ +#! /usr/bin/env bash + +bash ${DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh +source secrets-export.sh + +if [ -z "${RUN_WITH_MONGOCRYPTD}" ]; then + # Set up crypt shared lib if we don't want to use mongocryptd + bash .evergreen/prepare-crypt-shared-lib.sh + source crypt_shared.sh + echo "CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH" +else + echo "CRYPT_SHARED_LIB_PATH not set; using mongocryptd" +fi diff --git a/.gitignore b/.gitignore index 687f037b0e7..2070e0189c1 100644 --- a/.gitignore +++ b/.gitignore @@ -104,3 +104,5 @@ mo-expansion.yml expansions.sh .drivers-tools/ + +crypt_shared.sh diff --git a/test/csfle-kms-providers.js b/test/csfle-kms-providers.js new file mode 100644 index 00000000000..97b8855a570 --- /dev/null +++ b/test/csfle-kms-providers.js @@ -0,0 +1,46 @@ +'use strict'; + +const csfleKMSProviders = { + aws: { + accessKeyId: process.env.FLE_AWS_KEY, + secretAccessKey: process.env.FLE_AWS_SECRET + }, + azure: { + tenantId: process.env.FLE_AZURE_TENANTID, + clientId: process.env.FLE_AZURE_CLIENTID, + clientSecret: process.env.FLE_AZURE_CLIENTSECRET + }, + gcp: { + email: process.env.FLE_GCP_EMAIL, + privateKey: process.env.FLE_GCP_PRIVATEKEY + }, + local: { + key: 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk' + }, + kmip: { + endpoint: 'localhost:5698' + } +}; + +function getCSFLEKMSProviders() { + return JSON.parse(JSON.stringify(csfleKMSProviders)); +} + +const keys = [ + 'FLE_AWS_KEY', + 'FLE_AWS_SECRET', + 'FLE_AZURE_TENANTID', + 'FLE_AZURE_CLIENTID', + 'FLE_AZURE_CLIENTSECRET', + 'FLE_GCP_EMAIL', + 'FLE_GCP_PRIVATEKEY' +]; + +const isInEnvironment = key => typeof process.env[key] === 'string' && process.env[key].length > 0; +const missingKeys = keys.filter(key => !isInEnvironment(key)).join(','); + +module.exports = { + getCSFLEKMSProviders, + kmsCredentialsPresent: missingKeys === '', + missingKeys +}; diff --git a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts index ec2e32462ac..80d1741c50d 100644 --- a/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts +++ b/test/integration/client-side-encryption/client_side_encryption.prose.06.corpus.test.ts @@ -7,6 +7,7 @@ import * as path from 'path'; // eslint-disable-next-line @typescript-eslint/no-restricted-imports import { ClientEncryption } from '../../../src/client-side-encryption/client_encryption'; +import { getCSFLEKMSProviders } from '../../csfle-kms-providers'; import { type MongoClient, WriteConcern } from '../../mongodb'; import { getEncryptExtraOptions } from '../../tools/utils'; @@ -25,17 +26,7 @@ describe('Client Side Encryption Prose Corpus Test', function () { }); } - const CSFLE_KMS_PROVIDERS = process.env.CSFLE_KMS_PROVIDERS; - const kmsProviders = CSFLE_KMS_PROVIDERS ? EJSON.parse(CSFLE_KMS_PROVIDERS) : {}; - kmsProviders.local = { - key: Buffer.from( - 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk', - 'base64' - ) - }; - kmsProviders.kmip = { - endpoint: 'localhost:5698' - }; + const kmsProviders = getCSFLEKMSProviders(); // TODO: build this into EJSON // TODO: make a custom chai assertion for this @@ -201,8 +192,8 @@ describe('Client Side Encryption Prose Corpus Test', function () { // Configure both objects with ``keyVaultNamespace`` set to ``keyvault.datakeys``. const tlsOptions = { kmip: { - tlsCAFile: process.env.KMIP_TLS_CA_FILE, - tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE, + tlsCertificateKeyFile: process.env.CSFLE_TLS_CLIENT_CERT_FILE } }; const extraOptions = getEncryptExtraOptions(); diff --git a/test/integration/client-side-encryption/client_side_encryption.prose.21.automatic_data_encryption_keys.test.ts b/test/integration/client-side-encryption/client_side_encryption.prose.21.automatic_data_encryption_keys.test.ts index 5295535e52e..f81ab73d982 100644 --- a/test/integration/client-side-encryption/client_side_encryption.prose.21.automatic_data_encryption_keys.test.ts +++ b/test/integration/client-side-encryption/client_side_encryption.prose.21.automatic_data_encryption_keys.test.ts @@ -4,7 +4,12 @@ import { expect } from 'chai'; import { ClientEncryption } from '../../../src/client-side-encryption/client_encryption'; /* eslint-disable @typescript-eslint/no-restricted-imports */ import { MongoCryptCreateEncryptedCollectionError } from '../../../src/client-side-encryption/errors'; -import { BSON, Collection, type Db, MongoServerError } from '../../mongodb'; +import { + getCSFLEKMSProviders, + kmsCredentialsPresent, + missingKeys +} from '../../csfle-kms-providers'; +import { Collection, type Db, MongoServerError } from '../../mongodb'; const metadata: MongoDBMetadataUI = { requires: { @@ -33,14 +38,15 @@ describe('21. Automatic Data Encryption Keys', () => { beforeEach(async function () { client = this.configuration.newClient(); - if (typeof process.env.CSFLE_KMS_PROVIDERS !== 'string') { + if (!kmsCredentialsPresent) { if (this.currentTest) { - this.currentTest.skipReason = 'This test requires env CSFLE_KMS_PROVIDERS to be set'; + this.currentTest.skipReason = + 'This test requires FLE environment variables. Missing keys: ' + missingKeys; } return this.currentTest?.skip(); } - const { aws, local } = BSON.EJSON.parse(process.env.CSFLE_KMS_PROVIDERS); + const { aws, local } = getCSFLEKMSProviders(); clientEncryption = new ClientEncryption(client, { keyVaultClient: client, diff --git a/test/integration/client-side-encryption/client_side_encryption.prose.22.range_explicit_encryption.test.ts b/test/integration/client-side-encryption/client_side_encryption.prose.22.range_explicit_encryption.test.ts index a650ff19306..3e23c685b43 100644 --- a/test/integration/client-side-encryption/client_side_encryption.prose.22.range_explicit_encryption.test.ts +++ b/test/integration/client-side-encryption/client_side_encryption.prose.22.range_explicit_encryption.test.ts @@ -8,11 +8,10 @@ import { Decimal128, type Document, Double, Long, type MongoClient } from '../.. import { ClientEncryption } from '../../../src/client-side-encryption/client_encryption'; /* eslint-disable @typescript-eslint/no-restricted-imports */ import { MongoCryptError } from '../../../src/client-side-encryption/errors'; +import { getCSFLEKMSProviders } from '../../csfle-kms-providers'; const getKmsProviders = () => { - const result = EJSON.parse(process.env.CSFLE_KMS_PROVIDERS || '{}') as unknown as { - local: unknown; - }; + const result = getCSFLEKMSProviders(); return { local: result.local }; }; diff --git a/test/integration/client-side-encryption/client_side_encryption.prose.23.range_encryption_defaults.test.ts b/test/integration/client-side-encryption/client_side_encryption.prose.23.range_encryption_defaults.test.ts index f1fa7404ed4..ea4041f746b 100644 --- a/test/integration/client-side-encryption/client_side_encryption.prose.23.range_encryption_defaults.test.ts +++ b/test/integration/client-side-encryption/client_side_encryption.prose.23.range_encryption_defaults.test.ts @@ -2,7 +2,8 @@ import { expect } from 'chai'; /* eslint-disable @typescript-eslint/no-restricted-imports */ import { ClientEncryption } from '../../../src/client-side-encryption/client_encryption'; -import { type Binary, EJSON, Int32, Long } from '../../mongodb'; +import { getCSFLEKMSProviders } from '../../csfle-kms-providers'; +import { type Binary, Int32, Long } from '../../mongodb'; const metaData: MongoDBMetadataUI = { requires: { @@ -17,10 +18,8 @@ const metaData: MongoDBMetadataUI = { } }; -const getKmsProviders = (): { local: { key: string } } => { - const result = EJSON.parse(process.env.CSFLE_KMS_PROVIDERS || '{}') as unknown as { - local: { key: string }; - }; +const getKmsProviders = (): { local: { key: Buffer } } => { + const result = getCSFLEKMSProviders(); return { local: result.local }; }; diff --git a/test/integration/client-side-encryption/client_side_encryption.prose.test.js b/test/integration/client-side-encryption/client_side_encryption.prose.test.js index b9a707ef895..adbc8b4964b 100644 --- a/test/integration/client-side-encryption/client_side_encryption.prose.test.js +++ b/test/integration/client-side-encryption/client_side_encryption.prose.test.js @@ -19,9 +19,10 @@ const { ClientEncryption } = require('../../../src/client-side-encryption/client const { ClientSideEncryptionFilter } = require('../../tools/runner/filters/client_encryption_filter'); +const { getCSFLEKMSProviders } = require('../../csfle-kms-providers'); const getKmsProviders = (localKey, kmipEndpoint, azureEndpoint, gcpEndpoint) => { - const result = BSON.EJSON.parse(process.env.CSFLE_KMS_PROVIDERS || '{}'); + const result = getCSFLEKMSProviders(); if (localKey) { result.local = { key: localKey }; } @@ -863,8 +864,8 @@ describe('Client Side Encryption Prose Tests', metadata, function () { kmsProviders: customKmsProviders, tlsOptions: { kmip: { - tlsCAFile: process.env.KMIP_TLS_CA_FILE, - tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE, + tlsCertificateKeyFile: process.env.CSFLE_TLS_CLIENT_CERT_FILE } }, extraOptions: getEncryptExtraOptions() @@ -875,8 +876,8 @@ describe('Client Side Encryption Prose Tests', metadata, function () { kmsProviders: invalidKmsProviders, tlsOptions: { kmip: { - tlsCAFile: process.env.KMIP_TLS_CA_FILE, - tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE, + tlsCertificateKeyFile: process.env.CSFLE_TLS_CLIENT_CERT_FILE } }, extraOptions: getEncryptExtraOptions() @@ -1371,16 +1372,16 @@ describe('Client Side Encryption Prose Tests', metadata, function () { beforeEach(async function () { const tlsCaOptions = { aws: { - tlsCAFile: process.env.KMIP_TLS_CA_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE }, azure: { - tlsCAFile: process.env.KMIP_TLS_CA_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE }, gcp: { - tlsCAFile: process.env.KMIP_TLS_CA_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE }, kmip: { - tlsCAFile: process.env.KMIP_TLS_CA_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE } }; const clientNoTlsOptions = { @@ -1394,20 +1395,20 @@ describe('Client Side Encryption Prose Tests', metadata, function () { kmsProviders: getKmsProviders(null, null, '127.0.0.1:8002', '127.0.0.1:8002'), tlsOptions: { aws: { - tlsCAFile: process.env.KMIP_TLS_CA_FILE, - tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE, + tlsCertificateKeyFile: process.env.CSFLE_TLS_CLIENT_CERT_FILE }, azure: { - tlsCAFile: process.env.KMIP_TLS_CA_FILE, - tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE, + tlsCertificateKeyFile: process.env.CSFLE_TLS_CLIENT_CERT_FILE }, gcp: { - tlsCAFile: process.env.KMIP_TLS_CA_FILE, - tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE, + tlsCertificateKeyFile: process.env.CSFLE_TLS_CLIENT_CERT_FILE }, kmip: { - tlsCAFile: process.env.KMIP_TLS_CA_FILE, - tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE, + tlsCertificateKeyFile: process.env.CSFLE_TLS_CLIENT_CERT_FILE } }, extraOptions: getEncryptExtraOptions() @@ -1752,32 +1753,32 @@ describe('Client Side Encryption Prose Tests', metadata, function () { }, tlsOptions: { 'aws:no_client_cert': { - tlsCAFile: process.env.KMIP_TLS_CA_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE }, 'azure:no_client_cert': { - tlsCAFile: process.env.KMIP_TLS_CA_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE }, 'gcp:no_client_cert': { - tlsCAFile: process.env.KMIP_TLS_CA_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE }, 'kmip:no_client_cert': { - tlsCAFile: process.env.KMIP_TLS_CA_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE }, 'aws:with_tls': { - tlsCAFile: process.env.KMIP_TLS_CA_FILE, - tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE, + tlsCertificateKeyFile: process.env.CSFLE_TLS_CLIENT_CERT_FILE }, 'azure:with_tls': { - tlsCAFile: process.env.KMIP_TLS_CA_FILE, - tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE, + tlsCertificateKeyFile: process.env.CSFLE_TLS_CLIENT_CERT_FILE }, 'gcp:with_tls': { - tlsCAFile: process.env.KMIP_TLS_CA_FILE, - tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE, + tlsCertificateKeyFile: process.env.CSFLE_TLS_CLIENT_CERT_FILE }, 'kmip:with_tls': { - tlsCAFile: process.env.KMIP_TLS_CA_FILE, - tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE, + tlsCertificateKeyFile: process.env.CSFLE_TLS_CLIENT_CERT_FILE } }, keyVaultNamespace: 'db.keys' @@ -2354,8 +2355,8 @@ describe('Client Side Encryption Prose Tests', metadata, function () { kmsProviders: getKmsProviders(), tlsOptions: { kmip: { - tlsCAFile: process.env.KMIP_TLS_CA_FILE, - tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE, + tlsCertificateKeyFile: process.env.CSFLE_TLS_CLIENT_CERT_FILE } }, extraOptions: getEncryptExtraOptions(), @@ -2379,8 +2380,8 @@ describe('Client Side Encryption Prose Tests', metadata, function () { kmsProviders: getKmsProviders(), tlsOptions: { kmip: { - tlsCAFile: process.env.KMIP_TLS_CA_FILE, - tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE, + tlsCertificateKeyFile: process.env.CSFLE_TLS_CLIENT_CERT_FILE } }, extraOptions: getEncryptExtraOptions(), diff --git a/test/integration/client-side-encryption/driver.test.ts b/test/integration/client-side-encryption/driver.test.ts index a7c1e617c2a..2b0f6447f1a 100644 --- a/test/integration/client-side-encryption/driver.test.ts +++ b/test/integration/client-side-encryption/driver.test.ts @@ -1,4 +1,4 @@ -import { type Binary, EJSON, UUID } from 'bson'; +import { UUID } from 'bson'; import { expect } from 'chai'; import * as crypto from 'crypto'; import * as sinon from 'sinon'; @@ -6,6 +6,7 @@ import { setTimeout } from 'timers/promises'; // eslint-disable-next-line @typescript-eslint/no-restricted-imports import { ClientEncryption } from '../../../src/client-side-encryption/client_encryption'; +import { getCSFLEKMSProviders } from '../../csfle-kms-providers'; import { BSON, type Collection, @@ -37,12 +38,8 @@ const metadata: MongoDBMetadataUI = { } }; -const getLocalKmsProvider = (): { local: { key: Binary } } => { - const { local } = EJSON.parse(process.env.CSFLE_KMS_PROVIDERS || '{}') as { - local: { key: Binary }; - [key: string]: unknown; - }; - +const getLocalKmsProvider = (): { local: { key: Buffer } } => { + const { local } = getCSFLEKMSProviders(); return { local }; }; @@ -53,41 +50,6 @@ describe('Client Side Encryption Functional', function () { const keyVaultCollName = 'datakeys'; const keyVaultNamespace = `${keyVaultDbName}.${keyVaultCollName}`; - it('CSFLE_KMS_PROVIDERS should be valid EJSON', function () { - const CSFLE_KMS_PROVIDERS = process.env.CSFLE_KMS_PROVIDERS; - if (typeof CSFLE_KMS_PROVIDERS === 'string') { - /** - * The shape of CSFLE_KMS_PROVIDERS is as follows: - * - * ```ts - * interface CSFLE_kms_providers { - * aws: { - * accessKeyId: string; - * secretAccessKey: string; - * }; - * azure: { - * tenantId: string; - * clientId: string; - * clientSecret: string; - * }; - * gcp: { - * email: string; - * privateKey: string; - * }; - * local: { - * // EJSON handle converting this, its actually the canonical -> { $binary: { base64: string; subType: string } } - * // **NOTE**: The dollar sign has to be escaped when using this as an ENV variable - * key: Binary; - * } - * } - * ``` - */ - expect(() => EJSON.parse(CSFLE_KMS_PROVIDERS)).to.not.throw(SyntaxError); - } else { - this.skip(); - } - }); - describe('Collection', metadata, function () { describe('#bulkWrite()', metadata, function () { context('when encryption errors', function () { @@ -640,10 +602,8 @@ describe('Range Explicit Encryption with JS native types', function () { } }; - const getKmsProviders = (): { local: { key: string } } => { - const result = EJSON.parse(process.env.CSFLE_KMS_PROVIDERS || '{}') as unknown as { - local: { key: string }; - }; + const getKmsProviders = (): { local: { key: Buffer } } => { + const result = getCSFLEKMSProviders(); return { local: result.local }; }; @@ -1135,7 +1095,7 @@ describe('CSOT', function () { }; beforeEach(async function () { - local_key = { local: EJSON.parse(process.env.CSFLE_KMS_PROVIDERS).local }; + local_key = { local: getCSFLEKMSProviders().local }; client = this.configuration.newClient({ timeoutMS }); await client.connect(); await client.db('keyvault').createCollection('datakeys'); diff --git a/test/integration/client-side-operations-timeout/client_side_operations_timeout.unit.test.ts b/test/integration/client-side-operations-timeout/client_side_operations_timeout.unit.test.ts index 3515aaad921..4ca8e32f913 100644 --- a/test/integration/client-side-operations-timeout/client_side_operations_timeout.unit.test.ts +++ b/test/integration/client-side-operations-timeout/client_side_operations_timeout.unit.test.ts @@ -18,6 +18,7 @@ import { CSOTTimeoutContext, type MongoClient, MongoOperationTimeoutError, + ObjectId, Timeout, TimeoutContext, Topology @@ -195,11 +196,10 @@ describe('CSOT spec unit tests', function () { { autoEncryption: { extraOptions: { - mongocryptdBypassSpawn: true, - mongocryptdURI: 'mongodb://localhost:27017/db?serverSelectionTimeoutMS=1000', + mongocryptdURI: 'mongodb://localhost:27020/db?serverSelectionTimeoutMS=2000', mongocryptdSpawnArgs: [ - '--pidfilepath=bypass-spawning-mongocryptd.pid', - '--port=27017' + `--pidfilepath=${new ObjectId().toHexString()}.pid`, + '--port=27020' ] }, keyVaultNamespace: 'admin.datakeys', @@ -211,6 +211,7 @@ describe('CSOT spec unit tests', function () { timeoutMS } ); + await encryptedClient.connect(); const stub = sinon diff --git a/test/integration/node-specific/client_close.test.ts b/test/integration/node-specific/client_close.test.ts index 0743cd211db..d9b7683cb02 100644 --- a/test/integration/node-specific/client_close.test.ts +++ b/test/integration/node-specific/client_close.test.ts @@ -1,4 +1,5 @@ /* eslint-disable @typescript-eslint/no-empty-function */ +import { getCSFLEKMSProviders } from '../../csfle-kms-providers'; import { type TestConfiguration } from '../../tools/runner/config'; import { runScriptAndGetProcessInfo } from './resource_tracking_script_builder'; @@ -497,7 +498,7 @@ describe.skip('MongoClient.close() Integration', () => { async function run({ MongoClient, uri, expect, mongodb }) { const infiniteFile = '/dev/zero'; - const kmsProviders = mongodb.BSON.EJSON.parse(process.env.CSFLE_KMS_PROVIDERS); + const kmsProviders = getCSFLEKMSProviders(); const masterKey = { region: 'us-east-1', key: 'arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0' diff --git a/test/readme.md b/test/readme.md index 5dcfc4883cc..81c2068bd2b 100644 --- a/test/readme.md +++ b/test/readme.md @@ -31,7 +31,6 @@ about the types of tests and how to run them. - [Serverless](#serverless) - [Load Balanced](#load-balanced) - [Client-Side Field-Level Encryption (CSFLE)](#client-side-field-level-encryption-csfle) - - [KMIP FLE support tests](#kmip-fle-support-tests) - [Deployed KMS Tests](#deployed-kms-tests) - [Azure KMS](#azure-kms) - [GCP KMS](#gcp-kms) @@ -501,143 +500,39 @@ The following steps will walk you through how to start and test a load balancer. The following steps will walk you through how to run the tests for CSFLE. 1. Install [MongoDB Client Encryption][npm-csfle] if you haven't already: - ```sh - npm install mongodb-client-encryption - ``` - > **Note:** if developing changes in `mongodb-client-encryption`, - you can link it locally using `etc/tooling/fle.sh`. - -1. Create the following environment variables using a command like: - ```sh - export AWS_REGION="us-east-1" - ``` - > **Note:** MongoDB employees can pull these values from the Evergreen project's configuration. - - | Variable Name | Description | - | ----------------------- | ------------------------------------------------------------------------------------------- | - | `AWS_ACCESS_KEY_ID` | The AWS access key ID used to generate KMS messages | - | `AWS_SECRET_ACCESS_KEY` | The AWS secret access key used to generate KMS messages | - | `AWS_REGION` | The AWS region where the KMS resides (e.g., `us-east-1`) | - | `AWS_CMK_ID` | The Customer Master Key for the KMS | - | `CSFLE_KMS_PROVIDERS` | The raw EJSON description of the KMS providers. An example of the format is provided below. | - | `KMIP_TLS_CA_FILE` | /path/to/mongodb-labs/drivers-evergreen-tools/.evergreen/x509gen/ca.pem | - | `KMIP_TLS_CERT_FILE` | /path/to/mongodb-labs/drivers-evergreen-tools/.evergreen/x509gen/client.pem | - - The value of the `CSFLE_KMS_PROVIDERS` variable will have the following format: - - ``` - interface CSFLE_kms_providers { - aws: { - accessKeyId: string; - secretAccessKey: string; - }; - azure: { - tenantId: string; - clientId: string; - clientSecret: string; - }; - gcp: { - email: string; - privateKey: string; - }; - local: { - // EJSON handle converting this, its actually the canonical -> { $binary: { base64: string; subType: string } } - // **NOTE**: The dollar sign has to be escaped when using this as an ENV variable - key: Binary; - } - } - ``` -1. Start the KMIP servers: - - ```sh - DRIVERS_TOOLS="/path/to/mongodb-labs/drivers-evergreen-tools" .evergreen/run-kms-servers.sh - ``` - -1. Ensure default `~/.aws/config` is present: - - ``` - [default] - aws_access_key_id=AWS_ACCESS_KEY_ID - aws_secret_access_key=AWS_SECRET_ACCESS_KEY - ``` - -1. Set temporary AWS credentials - - ``` - source /path/to/mongodb-labs/drivers-evergreen-tools/.evergreen/csfle/activate-kmstlsvenv.sh - source /path/to/mongodb-labs/drivers-evergreen-tools/.evergreen/csfle/set-temp-creds.sh - ``` +```bash +npm install mongodb-client-encryption +``` +> [!NOTE] +> If developing changes in `mongodb-client-encryption`, you can link it locally using `etc/tooling/fle.sh`. - Alternatively, for fish users, the following script can be substituted for `set-temp-creds.sh`: +2. Load FLE credentials and download crypt_shared - ```fish - function set_aws_creds - set PYTHON_SCRIPT "\ - import boto3 - client = boto3.client('sts') - credentials = client.get_session_token()['Credentials'] - print (credentials['AccessKeyId'] + ' ' + credentials['SecretAccessKey'] + ' ' + credentials['SessionToken'])" +This must be run inside a bash or zsh shell. - echo $PYTHON_SCRIPT | python3 - - end +```bash +source .evergreen/setup-fle.sh +``` - set CREDS (set_aws_creds) +> [!NOTE] +> By default, `setup-fle.sh` installs crypt_shared. If you want to test with mongocryptd instead, set the RUN_WITH_MONGOCRYPTD environment variable before +> sourcing `setup-fle.sh`. - set CSFLE_AWS_TEMP_ACCESS_KEY_ID (echo $CREDS | awk '{print $1}') - set CSFLE_AWS_TEMP_SECRET_ACCESS_KEY (echo $CREDS | awk '{print $2}') - set CSFLE_AWS_TEMP_SESSION_TOKEN (echo $CREDS | awk '{print $3}') - set -e CREDS - ``` +3. Start the KMS and KMIP servers: -1. Run the functional tests: - ```sh - npm run check:test - ``` - - The output of the tests will include sections like "Client-Side Encryption Corpus", "Client-Side Encryption Functional", "Client-Side Encryption Prose Tests", and "Client-Side Encryption". - - To run the functional tests using the crypt shared library instead of `mongocryptd`, download the appropriate version of the crypt shared library for the enterprise server version [here](https://www.mongodb.com/download-center/enterprise/releases) and then set the location of it in the environment variable `CRYPT_SHARED_LIB_PATH`. +```bash +bash .evergreen/run-kms-servers.sh +bash .evergreen/run-kmip-server.sh & +``` -#### KMIP FLE support tests +4. Run the functional tests: +```bash +export TEST_CSFLE=true +npm run check:test +``` -1. Install `virtualenv`: - ```sh - pip install virtualenv - ``` -2. Source the `./activate-kmstlsvenv.sh` script in driver evergreen tools `.evergreen/csfle/activate-kmstlsvenv.sh` - - This will install all the dependencies needed to run a Python kms_kmip simulated server -3. In four separate terminals, launch the following: - ```sh - ./kmstlsvenv/bin/python3 -u kms_kmip_server.py` # by default it always runs on port 5698 - ``` - ```sh - ./kmstlsvenv/bin/python3 -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 - ``` - ```sh - ./kmstlsvenv/bin/python3 -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 - ``` - ```sh - ./kmstlsvenv/bin/python3 -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert - ``` -4. Set the following environment variables: - ```sh - export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem" - export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem" - ``` -5. Install the FLE lib: - ```sh - npm i --no-save mongodb-client-encryption - ``` -6. Launch a MongoDB server -7. Run the full suite: - ```sh - npm run check:test - ``` - or more specifically - ```sh - npx mocha --config test/mocha_mongodb.json test/integration/client-side-encryption/ - ``` +The output of the tests will include sections like "Client-Side Encryption Corpus", "Client-Side Encryption Functional", "Client-Side Encryption Prose Tests", and "Client-Side Encryption". ### Deployed KMS Tests diff --git a/test/tools/runner/config.ts b/test/tools/runner/config.ts index ed1510505b5..505f6a85b3e 100644 --- a/test/tools/runner/config.ts +++ b/test/tools/runner/config.ts @@ -60,7 +60,6 @@ export class TestConfiguration { clientSideEncryption: { enabled: boolean; mongodbClientEncryption: any; - CSFLE_KMS_PROVIDERS: string | undefined; version: string; libmongocrypt: string | null; }; @@ -199,7 +198,6 @@ export class TestConfiguration { newClient(urlOrQueryOptions?: string | Record, serverOptions?: MongoClientOptions) { serverOptions = Object.assign({}, getEnvironmentalOptions(), serverOptions); - // Support MongoClient constructor form (url, options) for `newClient`. if (typeof urlOrQueryOptions === 'string') { if (Reflect.has(serverOptions, 'host') || Reflect.has(serverOptions, 'port')) { diff --git a/test/tools/runner/filters/client_encryption_filter.ts b/test/tools/runner/filters/client_encryption_filter.ts index b3a407d3a6a..79983816d77 100644 --- a/test/tools/runner/filters/client_encryption_filter.ts +++ b/test/tools/runner/filters/client_encryption_filter.ts @@ -3,6 +3,7 @@ import { dirname, resolve } from 'path'; import * as process from 'process'; import { satisfies } from 'semver'; +import { kmsCredentialsPresent } from '../../../csfle-kms-providers'; import { type MongoClient } from '../../../mongodb'; import { Filter } from './filter'; @@ -24,10 +25,7 @@ export class ClientSideEncryptionFilter extends Filter { static version = null; static libmongocrypt: string | null = null; - csfleKMSProviders = null; - override async initializeFilter(client: MongoClient, context: Record) { - this.csfleKMSProviders = process.env.CSFLE_KMS_PROVIDERS; let mongodbClientEncryption; try { // eslint-disable-next-line @typescript-eslint/no-require-imports @@ -48,13 +46,12 @@ export class ClientSideEncryptionFilter extends Filter { ) ).version; - this.enabled = !!(this.csfleKMSProviders && mongodbClientEncryption); + this.enabled = !!(kmsCredentialsPresent && mongodbClientEncryption); // Adds these fields onto the context so that they can be reused by tests context.clientSideEncryption = { enabled: this.enabled, mongodbClientEncryption, - CSFLE_KMS_PROVIDERS: this.csfleKMSProviders, version: ClientSideEncryptionFilter.version, libmongocrypt: ClientSideEncryptionFilter.libmongocrypt }; @@ -76,15 +73,12 @@ export class ClientSideEncryptionFilter extends Filter { // TODO(NODE-3401): unskip csfle tests on windows if (process.env.TEST_CSFLE && process.platform !== 'win32') { - if (!this.csfleKMSProviders) { - throw new Error('FLE tests must run, but no KMS providers were set in the environment.'); - } if (ClientSideEncryptionFilter.version == null) { throw new Error('FLE tests must run, but mongodb client encryption was not installed.'); } } - if (this.csfleKMSProviders == null) return 'Test requires FLE environment variables.'; + if (!kmsCredentialsPresent) return 'Test requires FLE kms credentials'; if (ClientSideEncryptionFilter.version == null) return 'Test requires mongodb-client-encryption to be installed.'; diff --git a/test/tools/spec-runner/index.js b/test/tools/spec-runner/index.js index 42ea3b126b1..f312990137c 100644 --- a/test/tools/spec-runner/index.js +++ b/test/tools/spec-runner/index.js @@ -16,6 +16,7 @@ const { } = require('../../mongodb'); const { isAnyRequirementSatisfied } = require('../unified-spec-runner/unified-utils'); const { ClientSideEncryptionFilter } = require('../runner/filters/client_encryption_filter'); +const { getCSFLEKMSProviders } = require('../../csfle-kms-providers'); // Promise.try alternative https://stackoverflow.com/questions/60624081/promise-try-without-bluebird/60624164?noredirect=1#comment107255389_60624164 function promiseTry(callback) { @@ -57,7 +58,7 @@ function translateClientOptions(options) { } if (options.autoEncryptOpts.kmsProviders) { - const kmsProviders = EJSON.parse(process.env.CSFLE_KMS_PROVIDERS || 'NOT_PROVIDED'); + const kmsProviders = getCSFLEKMSProviders(); if (options.autoEncryptOpts.kmsProviders.local) { kmsProviders.local = options.autoEncryptOpts.kmsProviders.local; } @@ -83,8 +84,8 @@ function translateClientOptions(options) { }; options.autoEncryption.tlsOptions = { kmip: { - tlsCAFile: process.env.KMIP_TLS_CA_FILE, - tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE, + tlsCertificateKeyFile: process.env.CSFLE_TLS_CLIENT_CERT_FILE } }; } @@ -93,8 +94,8 @@ function translateClientOptions(options) { kmsProviders['local:name2'] = options.autoEncryptOpts.kmsProviders['local:name2']; options.autoEncryption.tlsOptions = { 'local:name2': { - tlsCAFile: process.env.KMIP_TLS_CA_FILE, - tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE + tlsCAFile: process.env.CSFLE_TLS_CA_FILE, + tlsCertificateKeyFile: process.env.CSFLE_TLS_CLIENT_CERT_FILE } }; } diff --git a/test/tools/unified-spec-runner/unified-utils.ts b/test/tools/unified-spec-runner/unified-utils.ts index 25a5115a6d5..56c9a54e92e 100644 --- a/test/tools/unified-spec-runner/unified-utils.ts +++ b/test/tools/unified-spec-runner/unified-utils.ts @@ -1,4 +1,3 @@ -import { EJSON } from 'bson'; import { AssertionError, expect } from 'chai'; import ConnectionString from 'mongodb-connection-string-url'; import { gte as semverGte, lte as semverLte } from 'semver'; @@ -6,6 +5,7 @@ import { isDeepStrictEqual } from 'util'; /* eslint-disable @typescript-eslint/no-restricted-imports */ import { ClientEncryption } from '../../../src/client-side-encryption/client_encryption'; +import { getCSFLEKMSProviders } from '../../csfle-kms-providers'; import { type AutoEncryptionOptions, type CollectionOptions, @@ -247,9 +247,8 @@ export function getMatchingEventCount(event, client, entities): number { /** * parses the process.env for three required environment variables * - * - CSFLE_KMS_PROVIDERS - * - KMIP_TLS_CA_FILE - * - KMIP_TLS_CERT_FILE + * - CSFLE_TLS_CA_FILE + * - CSFLE_TLS_CLIENT_CERT_FILE * * @throws if any required environment variable is undefined, or if we are unable to * parse the CLSFE_KMS_PROVIDERS from the environment @@ -258,38 +257,26 @@ export function getCSFLETestDataFromEnvironment(environment: Record