Skip to content

Commit 579219c

Browse files
baileympearsonbaileympearson
authored
refactor(NODE-5419): move csfle source and tests into the driver (mongodb#3770)
1 parent 7a91714 commit 579219c

File tree

104 files changed

+4649
-4937
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

104 files changed

+4649
-4937
lines changed

‎.eslintignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,6 @@
22
lib
33
test/disabled
44
!etc/docs
5+
6+
src/client-side-encryption
7+
test/unit/client-side-encryption

‎.evergreen/config.in.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,8 @@ functions:
196196
export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
197197
export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
198198
export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
199+
export AWS_REGION='${AWS_REGION}'
200+
export AWS_CMK_ID='${AWS_CMK_ID}'
199201
export AWS_DEFAULT_REGION='us-east-1'
200202
export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
201203
export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem"
@@ -250,6 +252,8 @@ functions:
250252
export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
251253
export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
252254
export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
255+
export AWS_REGION='${AWS_REGION}'
256+
export AWS_CMK_ID='${AWS_CMK_ID}'
253257
export AWS_DEFAULT_REGION='us-east-1'
254258
export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
255259
export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem"
@@ -557,6 +561,8 @@ functions:
557561
cat <<EOT > prepare_client_encryption.sh
558562
export CLIENT_ENCRYPTION='${CLIENT_ENCRYPTION}'
559563
export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
564+
export AWS_REGION='${AWS_REGION}'
565+
export AWS_CMK_ID='${AWS_CMK_ID}'
560566
export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
561567
export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
562568
export CSFLE_GIT_REF='${CSFLE_GIT_REF}'
@@ -974,6 +980,8 @@ functions:
974980
export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
975981
export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
976982
export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
983+
export AWS_REGION='${AWS_REGION}'
984+
export AWS_CMK_ID='${AWS_CMK_ID}'
977985
export CSFLE_GIT_REF='${CSFLE_GIT_REF}'
978986
export CDRIVER_GIT_REF='${CDRIVER_GIT_REF}'
979987
EOT

‎.evergreen/config.yml‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,8 @@ functions:
165165
export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
166166
export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
167167
export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
168+
export AWS_REGION='${AWS_REGION}'
169+
export AWS_CMK_ID='${AWS_CMK_ID}'
168170
export AWS_DEFAULT_REGION='us-east-1'
169171
export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
170172
export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem"
@@ -218,6 +220,8 @@ functions:
218220
export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
219221
export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
220222
export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
223+
export AWS_REGION='${AWS_REGION}'
224+
export AWS_CMK_ID='${AWS_CMK_ID}'
221225
export AWS_DEFAULT_REGION='us-east-1'
222226
export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
223227
export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem"
@@ -501,6 +505,8 @@ functions:
501505
cat <<EOT > prepare_client_encryption.sh
502506
export CLIENT_ENCRYPTION='${CLIENT_ENCRYPTION}'
503507
export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
508+
export AWS_REGION='${AWS_REGION}'
509+
export AWS_CMK_ID='${AWS_CMK_ID}'
504510
export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
505511
export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
506512
export CSFLE_GIT_REF='${CSFLE_GIT_REF}'
@@ -921,6 +927,8 @@ functions:
921927
export CSFLE_KMS_PROVIDERS='${CSFLE_KMS_PROVIDERS}'
922928
export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
923929
export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
930+
export AWS_REGION='${AWS_REGION}'
931+
export AWS_CMK_ID='${AWS_CMK_ID}'
924932
export CSFLE_GIT_REF='${CSFLE_GIT_REF}'
925933
export CDRIVER_GIT_REF='${CDRIVER_GIT_REF}'
926934
EOT
@@ -2668,7 +2676,7 @@ tasks:
26682676
- func: bootstrap kms servers
26692677
- func: run custom csfle tests
26702678
vars:
2671-
CSFLE_GIT_REF: c56c70340093070b1ef5c8a28190187eea21a6e9
2679+
CSFLE_GIT_REF: 974a4614f8c1c3786e5e39fa63568d83f4f69ebd
26722680
- name: run-custom-csfle-tests-5.0-master
26732681
tags:
26742682
- run-custom-dependency-tests
@@ -2698,7 +2706,7 @@ tasks:
26982706
- func: bootstrap kms servers
26992707
- func: run custom csfle tests
27002708
vars:
2701-
CSFLE_GIT_REF: c56c70340093070b1ef5c8a28190187eea21a6e9
2709+
CSFLE_GIT_REF: 974a4614f8c1c3786e5e39fa63568d83f4f69ebd
27022710
- name: run-custom-csfle-tests-rapid-master
27032711
tags:
27042712
- run-custom-dependency-tests
@@ -2728,7 +2736,7 @@ tasks:
27282736
- func: bootstrap kms servers
27292737
- func: run custom csfle tests
27302738
vars:
2731-
CSFLE_GIT_REF: c56c70340093070b1ef5c8a28190187eea21a6e9
2739+
CSFLE_GIT_REF: 974a4614f8c1c3786e5e39fa63568d83f4f69ebd
27322740
- name: run-custom-csfle-tests-latest-master
27332741
tags:
27342742
- run-custom-dependency-tests
@@ -3646,7 +3654,6 @@ buildvariants:
36463654
- test-auth-ldap
36473655
- test-auth-oidc
36483656
- test-socks5
3649-
- test-socks5-csfle
36503657
- test-socks5-tls
36513658
- test-tls-support-latest
36523659
- test-tls-support-6.0
@@ -3697,7 +3704,6 @@ buildvariants:
36973704
- test-auth-ldap
36983705
- test-auth-oidc
36993706
- test-socks5
3700-
- test-socks5-csfle
37013707
- test-socks5-tls
37023708
- test-tls-support-latest
37033709
- test-tls-support-6.0

‎.evergreen/generate_evergreen_tasks.js‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -643,7 +643,7 @@ BUILD_VARIANTS.push({
643643

644644
const oneOffFuncAsTasks = [];
645645

646-
const FLE_PINNED_COMMIT = 'c56c70340093070b1ef5c8a28190187eea21a6e9';
646+
const FLE_PINNED_COMMIT = '974a4614f8c1c3786e5e39fa63568d83f4f69ebd';
647647

648648
for (const version of ['5.0', 'rapid', 'latest']) {
649649
for (const ref of [FLE_PINNED_COMMIT, 'master']) {
@@ -807,6 +807,13 @@ for (const variant of BUILD_VARIANTS.filter(
807807
variant.tasks = variant.tasks.filter(name => !['test-socks5'].includes(name));
808808
}
809809

810+
// TODO(NODE-5283): fix socks5 fle tests on node 20+
811+
for (const variant of BUILD_VARIANTS.filter(
812+
variant => variant.expansions && [20].includes(variant.expansions.NODE_LTS_VERSION)
813+
) ) {
814+
variant.tasks = variant.tasks.filter(name => !['test-socks5-csfle'].includes(name));
815+
}
816+
810817
const fileData = yaml.load(fs.readFileSync(`${__dirname}/config.in.yml`, 'utf8'));
811818
fileData.tasks = (fileData.tasks || [])
812819
.concat(BASE_TASKS)

‎.evergreen/run-azure-kms-tests.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ source ".evergreen/init-node-and-npm-env.sh"
99

1010
set -o xtrace
1111

12-
npm install mongodb-client-encryption
12+
npm install mongodb-client-encryption@alpha --force
1313

1414
export MONGODB_URI="mongodb://localhost:27017"
1515

‎.evergreen/run-custom-csfle-tests.sh‎

Lines changed: 6 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -55,54 +55,20 @@ pushd libmongocrypt/bindings/node
5555
npm install --production --ignore-scripts
5656
bash ./etc/build-static.sh
5757

58+
npm run rebuild # just in case this is necessary?
59+
60+
ls
61+
ls lib
62+
BINDINGS_DIR=$(pwd)
5863
popd # libmongocrypt/bindings/node
5964
popd # ../csfle-deps-tmp
6065

6166
# copy mongodb-client-encryption into driver's node_modules
62-
cp -R ../csfle-deps-tmp/libmongocrypt/bindings/node node_modules/mongodb-client-encryption
67+
npm link $BINDINGS_DIR
6368

6469
export MONGODB_URI=${MONGODB_URI}
6570
export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
6671
export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem"
6772
export TEST_CSFLE=true
6873

69-
set +o errexit # We want to run both test suites even if the first fails
7074
npm run check:csfle
71-
DRIVER_CSFLE_TEST_RESULT=$?
72-
set -o errexit
73-
74-
# Great! our drivers tests ran
75-
# there are tests inside the bindings repo that we also want to check
76-
77-
pushd ../csfle-deps-tmp/libmongocrypt/bindings/node
78-
79-
# a mongocryptd was certainly started by the driver tests,
80-
# let us let the bindings tests start their own
81-
killall mongocryptd || true
82-
83-
# only prod deps were installed earlier, install devDependencies here (except for mongodb!)
84-
npm install --ignore-scripts
85-
86-
# copy mongodb into CSFLE's node_modules
87-
rm -rf node_modules/mongodb
88-
cp -R "$ABS_PATH_TO_PATCH" node_modules/mongodb
89-
pushd node_modules/mongodb
90-
# lets be sure we have compiled TS since driver tests don't need to compile
91-
npm run build:ts
92-
popd # node_modules/mongodb
93-
94-
# this variable needs to be empty
95-
export MONGODB_NODE_SKIP_LIVE_TESTS=""
96-
# all of the below must be defined (as well as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
97-
export AWS_REGION="us-east-1"
98-
export AWS_CMK_ID="arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0"
99-
100-
npm test -- --colors
101-
102-
popd # ../csfle-deps-tmp/libmongocrypt/bindings/node
103-
104-
# Exit the script in a way that will show evergreen a pass or fail
105-
if [ $DRIVER_CSFLE_TEST_RESULT -ne 0 ]; then
106-
echo "Driver tests failed, look above for results"
107-
exit 1
108-
fi

‎.evergreen/run-gcp-kms-tests.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ source ".evergreen/init-node-and-npm-env.sh"
99

1010
set -o xtrace
1111

12-
npm install mongodb-client-encryption
12+
npm install mongodb-client-encryption@alpha --force
1313
npm install gcp-metadata
1414

1515
export MONGODB_URI="mongodb://localhost:27017"

‎.evergreen/run-serverless-tests.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ if [ -z ${MONGODB_URI+omitted} ]; then echo "MONGODB_URI is unset" && exit 1; fi
1010
if [ -z ${SERVERLESS_ATLAS_USER+omitted} ]; then echo "SERVERLESS_ATLAS_USER is unset" && exit 1; fi
1111
if [ -z ${SERVERLESS_ATLAS_PASSWORD+omitted} ]; then echo "SERVERLESS_ATLAS_PASSWORD is unset" && exit 1; fi
1212

13-
npm install mongodb-client-encryption
13+
npm install mongodb-client-encryption@alpha --force
1414

1515
npx mocha \
1616
--config test/mocha_mongodb.json \

‎.evergreen/run-socks5-tests.sh‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,27 @@ source "${PROJECT_DIRECTORY}/.evergreen/init-node-and-npm-env.sh"
55
set -o errexit # Exit the script with error if any of the commands fail
66
set -o xtrace # For debuggability, no external credentials are used here
77

8+
function setup_fle() {
9+
export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
10+
export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
11+
export CSFLE_KMS_PROVIDERS=${CSFLE_KMS_PROVIDERS}
12+
export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH}
13+
echo "csfle CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH"
14+
15+
set -o xtrace # Write all commands first to stderr
16+
set -o errexit # Exit the script with error if any of the commands fail
17+
18+
# Get access to the AWS temporary credentials:
19+
echo "adding temporary AWS credentials to environment"
20+
# CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN
21+
. "$DRIVERS_TOOLS"/.evergreen/csfle/set-temp-creds.sh
22+
23+
npm i --force mongodb-client-encryption@alpha
24+
export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
25+
export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem"
26+
export TEST_CSFLE=true
27+
}
28+
829
node -v
930

1031
PYTHON_BINARY=${PYTHON_BINARY:-python3}
@@ -32,9 +53,10 @@ fi
3253
"$PYTHON_BINARY" "$SOCKS5_SERVER_SCRIPT" --port 1080 --auth username:p4ssw0rd --map "127.0.0.1:12345 to $FIRST_HOST" &
3354
SOCKS5_PROXY_PID=$!
3455
if [[ $TEST_SOCKS5_CSFLE == "true" ]]; then
56+
setup_fle
3557
[ "$SSL" == "nossl" ] && [[ "$OSTYPE" == "linux-gnu"* ]] && \
3658
env MONGODB_URI='mongodb://127.0.0.1:12345/?proxyHost=127.0.0.1&proxyUsername=username&proxyPassword=p4ssw0rd' \
37-
bash "${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh"
59+
npm run check:csfle
3860
else
3961
env SOCKS5_CONFIG='["127.0.0.1",1080,"username","p4ssw0rd"]' npm run check:socks5
4062
fi
@@ -44,9 +66,10 @@ kill $SOCKS5_PROXY_PID
4466
"$PYTHON_BINARY" "$SOCKS5_SERVER_SCRIPT" --port 1081 --map "127.0.0.1:12345 to $FIRST_HOST" &
4567
SOCKS5_PROXY_PID=$!
4668
if [[ $TEST_SOCKS5_CSFLE == "true" ]]; then
69+
setup_fle
4770
[ "$SSL" == "nossl" ] && [[ "$OSTYPE" == "linux-gnu"* ]] && \
4871
env MONGODB_URI='mongodb://127.0.0.1:12345/?proxyHost=127.0.0.1&proxyPort=1081' \
49-
bash "${PROJECT_DIRECTORY}/.evergreen/run-custom-csfle-tests.sh"
72+
npm run check:csfle
5073
else
5174
env SOCKS5_CONFIG='["127.0.0.1",1081]' npm run check:socks5
5275
fi

‎.evergreen/run-tests.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ else
5252
source "$DRIVERS_TOOLS"/.evergreen/csfle/set-temp-creds.sh
5353
fi
5454

55-
npm install mongodb-client-encryption
55+
npm install mongodb-client-encryption@alpha --force
5656
npm install @mongodb-js/zstd
5757
npm install snappy
5858

0 commit comments

Comments
 (0)