Skip to content

Commit 7f1a88b

Browse files
kevinAlbskevinAlbs
authored
CDRIVER-4512 remove legacy shell from AWS test scripts (#1514)
* run `setup_secrets.sh` for AWS auth * use `aws_setup.sh` instead of legacy shell setup for AWS test cases * remove unused `url_encode` * make `*.sh` scripts readable and executable in drivers-evergreen-tools * add `secrets-export.sh` to .gitignore * remove outdated comment * remove unnecessary PATH modification * remove unused `mongodb_bin_dir` * add shellcheck directives Ignore `aws_setup.sh` since it is in drivers-evergreen-tools * add missing `LAMBDA` test case to test without a session token * migrate Ubuntu 18.04 AWS tasks to Ubuntu 20.04 * migrate 6.0+ tasks to Ubuntu 22.04 * enable tracing and print "aws_auth" trace logs This is intended to help diagnose spurious observed auth failures * replace legacy shell with `mongosh` in tutorial
1 parent 43644f8 commit 7f1a88b

File tree

11 files changed

+147
-158
lines changed

11 files changed

+147
-158
lines changed

‎.evergreen/config_generator/components/funcs/fetch_det.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,13 @@ class FetchDET(Function):
1717
''',
1818
),
1919

20+
# Make shell scripts executable.
21+
bash_exec(
22+
command_type=EvgCommandType.SETUP,
23+
working_dir="drivers-evergreen-tools",
24+
script='find .evergreen -type f -name "*.sh" -execdir chmod +rx "{}" \;',
25+
),
26+
2027
# python is used frequently enough by many tasks that it is worth
2128
# running find_python3 once here and reusing the result.
2229
bash_exec(

‎.evergreen/generated_configs/functions.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,14 @@ functions:
173173
if [[ ! -d drivers-evergreen-tools ]]; then
174174
git clone --depth=1 [email protected]:mongodb-labs/drivers-evergreen-tools.git
175175
fi
176+
- command: subprocess.exec
177+
type: setup
178+
params:
179+
binary: bash
180+
working_dir: drivers-evergreen-tools
181+
args:
182+
- -c
183+
- find .evergreen -type f -name "*.sh" -execdir chmod +rx "{}" \;
176184
- command: subprocess.exec
177185
type: setup
178186
params:

‎.evergreen/generated_configs/legacy-config.yml

Lines changed: 20 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -263,43 +263,29 @@ functions:
263263
./src/libmongoc/test-libmongoc --no-fork -l /mongohouse/* -d --skip-tests .evergreen/etc/skip-tests.txt
264264
unset RUN_MONGOHOUSE_TESTS
265265
run aws tests:
266+
- command: ec2.assume_role
267+
params:
268+
role_arn: ${aws_test_secrets_role}
266269
- command: shell.exec
267270
type: test
268271
params:
269-
silent: true
270272
working_dir: mongoc
273+
include_expansions_in_env:
274+
- AWS_ACCESS_KEY_ID
275+
- AWS_SECRET_ACCESS_KEY
276+
- AWS_SESSION_TOKEN
271277
shell: bash
272278
script: |-
273279
set -o errexit
274-
# Add AWS variables to a file.
275-
cat <<EOF > ../drivers-evergreen-tools/.evergreen/auth_aws/aws_e2e_setup.json
276-
{
277-
"iam_auth_ecs_account" : "${iam_auth_ecs_account}",
278-
"iam_auth_ecs_secret_access_key" : "${iam_auth_ecs_secret_access_key}",
279-
"iam_auth_ecs_account_arn": "arn:aws:iam::557821124784:user/authtest_fargate_user",
280-
"iam_auth_ecs_cluster": "${iam_auth_ecs_cluster}",
281-
"iam_auth_ecs_task_definition": "${iam_auth_ecs_task_definition}",
282-
"iam_auth_ecs_subnet_a": "${iam_auth_ecs_subnet_a}",
283-
"iam_auth_ecs_subnet_b": "${iam_auth_ecs_subnet_b}",
284-
"iam_auth_ecs_security_group": "${iam_auth_ecs_security_group}",
285-
"iam_auth_assume_aws_account" : "${iam_auth_assume_aws_account}",
286-
"iam_auth_assume_aws_secret_access_key" : "${iam_auth_assume_aws_secret_access_key}",
287-
"iam_auth_assume_role_name" : "${iam_auth_assume_role_name}",
288-
"iam_auth_ec2_instance_account" : "${iam_auth_ec2_instance_account}",
289-
"iam_auth_ec2_instance_secret_access_key" : "${iam_auth_ec2_instance_secret_access_key}",
290-
"iam_auth_ec2_instance_profile" : "${iam_auth_ec2_instance_profile}",
291-
"iam_auth_assume_web_role_name": "${iam_auth_assume_web_role_name}",
292-
"iam_web_identity_issuer": "${iam_web_identity_issuer}",
293-
"iam_web_identity_rsa_key": "${iam_web_identity_rsa_key}",
294-
"iam_web_identity_jwks_uri": "${iam_web_identity_jwks_uri}",
295-
"iam_web_identity_token_file": "${iam_web_identity_token_file}"
296-
}
297-
EOF
280+
pushd ../drivers-evergreen-tools/.evergreen/auth_aws
281+
./setup_secrets.sh drivers/aws_auth
282+
popd # ../drivers-evergreen-tools/.evergreen/auth_aws
298283
- command: shell.exec
299284
type: test
300285
params:
301286
working_dir: mongoc
302-
add_expansions_to_env: true
287+
include_expansions_in_env:
288+
- TESTCASE
303289
shell: bash
304290
script: |-
305291
set -o errexit
@@ -1800,9 +1786,9 @@ tasks:
18001786
export distro_id='${distro_id}' # Required by find_cmake_latest.
18011787
. .evergreen/scripts/find-cmake-latest.sh
18021788
cmake_binary="$(find_cmake_latest)"
1803-
# Compile test-awsauth. Disable unnecessary dependencies since test-awsauth is copied to a remote Ubuntu 18.04 ECS cluster for testing, which may not have all dependent libraries.
1789+
# Compile test-awsauth. Disable unnecessary dependencies since test-awsauth is copied to a remote Ubuntu 20.04 ECS cluster for testing, which may not have all dependent libraries.
18041790
export CC='${CC}'
1805-
"$cmake_binary" -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF -DENABLE_ZSTD=OFF -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF .
1791+
"$cmake_binary" -DENABLE_TRACING=ON -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF -DENABLE_ZSTD=OFF -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF .
18061792
"$cmake_binary" --build . --target test-awsauth
18071793
- func: upload-build
18081794
- name: test-aws-openssl-regular-latest
@@ -14412,23 +14398,23 @@ buildvariants:
1441214398
tasks:
1441314399
- debug-compile-sasl-openssl-static
1441414400
- .authentication-tests .asan
14415-
- name: aws-ubuntu1804
14416-
display_name: AWS Tests (Ubuntu 18.04)
14401+
- name: aws-ubuntu2004
14402+
display_name: AWS Tests (Ubuntu 20.04)
1441714403
expansions:
1441814404
CC: clang
14419-
run_on: ubuntu1804-small
14405+
run_on: ubuntu2004-small
1442014406
tasks:
1442114407
- debug-compile-aws
1442214408
- .test-aws .4.4
1442314409
- .test-aws .5.0
14424-
- .test-aws .6.0
14425-
- name: aws-ubuntu2004
14426-
display_name: AWS Tests (Ubuntu 20.04)
14410+
- name: aws-ubuntu2204
14411+
display_name: AWS Tests (Ubuntu 22.04)
1442714412
expansions:
1442814413
CC: clang
1442914414
run_on: ubuntu2004-small
1443014415
tasks:
1443114416
- debug-compile-aws
14417+
- .test-aws .6.0
1443214418
- .test-aws .7.0
1443314419
- .test-aws .latest
1443414420
- name: mongohouse

‎.evergreen/legacy_config_generator/evergreen_config_lib/functions.py

Lines changed: 14 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -196,37 +196,25 @@
196196
'''),
197197
)),
198198
('run aws tests', Function(
199-
shell_mongoc(r'''
200-
# Add AWS variables to a file.
201-
cat <<EOF > ../drivers-evergreen-tools/.evergreen/auth_aws/aws_e2e_setup.json
199+
# Assume role to get AWS secrets.
202200
{
203-
"iam_auth_ecs_account" : "${iam_auth_ecs_account}",
204-
"iam_auth_ecs_secret_access_key" : "${iam_auth_ecs_secret_access_key}",
205-
"iam_auth_ecs_account_arn": "arn:aws:iam::557821124784:user/authtest_fargate_user",
206-
"iam_auth_ecs_cluster": "${iam_auth_ecs_cluster}",
207-
"iam_auth_ecs_task_definition": "${iam_auth_ecs_task_definition}",
208-
"iam_auth_ecs_subnet_a": "${iam_auth_ecs_subnet_a}",
209-
"iam_auth_ecs_subnet_b": "${iam_auth_ecs_subnet_b}",
210-
"iam_auth_ecs_security_group": "${iam_auth_ecs_security_group}",
211-
"iam_auth_assume_aws_account" : "${iam_auth_assume_aws_account}",
212-
"iam_auth_assume_aws_secret_access_key" : "${iam_auth_assume_aws_secret_access_key}",
213-
"iam_auth_assume_role_name" : "${iam_auth_assume_role_name}",
214-
"iam_auth_ec2_instance_account" : "${iam_auth_ec2_instance_account}",
215-
"iam_auth_ec2_instance_secret_access_key" : "${iam_auth_ec2_instance_secret_access_key}",
216-
"iam_auth_ec2_instance_profile" : "${iam_auth_ec2_instance_profile}",
217-
"iam_auth_assume_web_role_name": "${iam_auth_assume_web_role_name}",
218-
"iam_web_identity_issuer": "${iam_web_identity_issuer}",
219-
"iam_web_identity_rsa_key": "${iam_web_identity_rsa_key}",
220-
"iam_web_identity_jwks_uri": "${iam_web_identity_jwks_uri}",
221-
"iam_web_identity_token_file": "${iam_web_identity_token_file}"
222-
}
223-
EOF
224-
''', silent=True),
201+
"command": "ec2.assume_role",
202+
"params": {
203+
"role_arn": "${aws_test_secrets_role}"
204+
}
205+
},
206+
207+
shell_mongoc(r'''
208+
pushd ../drivers-evergreen-tools/.evergreen/auth_aws
209+
./setup_secrets.sh drivers/aws_auth
210+
popd # ../drivers-evergreen-tools/.evergreen/auth_aws
211+
''', include_expansions_in_env=["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]),
212+
225213
shell_mongoc(r'''
226214
pushd ../drivers-evergreen-tools/.evergreen/auth_aws
227215
. ./activate-authawsvenv.sh
228216
popd # ../drivers-evergreen-tools/.evergreen/auth_aws
229217
bash .evergreen/scripts/run-aws-tests.sh
230-
''', add_expansions_to_env=True)
218+
''', include_expansions_in_env=["TESTCASE"])
231219
)),
232220
])

‎.evergreen/legacy_config_generator/evergreen_config_lib/tasks.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -926,9 +926,9 @@ def do_is_valid_combination(self) -> bool:
926926
export distro_id='${distro_id}' # Required by find_cmake_latest.
927927
. .evergreen/scripts/find-cmake-latest.sh
928928
cmake_binary="$(find_cmake_latest)"
929-
# Compile test-awsauth. Disable unnecessary dependencies since test-awsauth is copied to a remote Ubuntu 18.04 ECS cluster for testing, which may not have all dependent libraries.
929+
# Compile test-awsauth. Disable unnecessary dependencies since test-awsauth is copied to a remote Ubuntu 20.04 ECS cluster for testing, which may not have all dependent libraries.
930930
export CC='${CC}'
931-
"$cmake_binary" -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF -DENABLE_ZSTD=OFF -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF .
931+
"$cmake_binary" -DENABLE_TRACING=ON -DENABLE_SASL=OFF -DENABLE_SNAPPY=OFF -DENABLE_ZSTD=OFF -DENABLE_CLIENT_SIDE_ENCRYPTION=OFF .
932932
"$cmake_binary" --build . --target test-awsauth
933933
"""
934934
),

‎.evergreen/legacy_config_generator/evergreen_config_lib/variants.py

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -422,25 +422,25 @@ def days(n: int) -> int:
422422
],
423423
{"CC": "clang"},
424424
),
425+
# Run AWS tests for MongoDB 4.4 and 5.0 on Ubuntu 20.04. AWS setup scripts expect Ubuntu 20.04+. MongoDB 4.4 and 5.0 are not available on 22.04.
425426
Variant(
426-
"aws-ubuntu1804",
427-
"AWS Tests (Ubuntu 18.04)",
428-
"ubuntu1804-small",
427+
"aws-ubuntu2004",
428+
"AWS Tests (Ubuntu 20.04)",
429+
"ubuntu2004-small",
429430
[
430431
"debug-compile-aws",
431432
".test-aws .4.4",
432433
".test-aws .5.0",
433-
".test-aws .6.0",
434434
],
435435
{"CC": "clang"},
436436
),
437-
# Test 7.0+ with Ubuntu 20.04+ since MongoDB 7.0 no longer ships binaries for Ubuntu 18.04.
438437
Variant(
439-
"aws-ubuntu2004",
440-
"AWS Tests (Ubuntu 20.04)",
438+
"aws-ubuntu2204",
439+
"AWS Tests (Ubuntu 22.04)",
441440
"ubuntu2004-small",
442441
[
443442
"debug-compile-aws",
443+
".test-aws .6.0",
444444
".test-aws .7.0",
445445
".test-aws .latest",
446446
],

0 commit comments

Comments
 (0)