Skip to content

Commit ffe66af

Browse files
ambushworkambushwork
committed
RUM-11897: Add scripts to set/get Vault secrets & Migrate CI
1 parent eebab6b commit ffe66af

File tree

5 files changed

+281
-38
lines changed

5 files changed

+281
-38
lines changed

‎ci/pipelines/default-pipeline.yml‎

Lines changed: 48 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ stages:
1313
- notify
1414

1515
.snippets:
16+
source-secrets:
17+
- source ./ci/scripts/vault_config.sh
18+
- source ./ci/scripts/get-secret.sh
19+
1620
# macOS AMI will already have cmdline-tools installed
1721
install-android-api-components:
1822
- echo y | ~/android_sdk/cmdline-tools/latest/bin/sdkmanager --install "emulator"
@@ -39,12 +43,12 @@ stages:
3943
- if [[ "$exit_code" -ne 0 ]]; then exit 1; fi
4044
- exit 0
4145
set-publishing-credentials:
42-
- aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.gradle-properties --with-decryption --query "Parameter.Value" --out text >> ./gradle.properties
43-
- export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text)
44-
- export GPG_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text)
45-
- export CENTRAL_PUBLISHER_USERNAME=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.publishing.central_username --with-decryption --query "Parameter.Value" --out text)
46-
- export CENTRAL_PUBLISHER_PASSWORD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.publishing.central_password --with-decryption --query "Parameter.Value" --out text)
47-
- export GPG_PUBLIC_FINGERPRINT=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.signing.gpg_public_key --with-decryption --query "Parameter.Value" --out text | gpg --import --import-options show-only | grep -E -o -e "[A-F0-9]{40}")
46+
- get_secret $DD_ANDROID_SECRET__GRADLE_PROPERTIES > ./gradle.properties
47+
- export GPG_PRIVATE_KEY=$(get_secret $DD_ANDROID_SECRET__SIGNING_GPG_PRIVATE_KEY)
48+
- export GPG_PASSWORD=$(get_secret $DD_ANDROID_SECRET__SIGNING_GPG_PASSPHRASE)
49+
- export CENTRAL_PUBLISHER_USERNAME=$(get_secret $DD_ANDROID_SECRET__PUBLISHING_CENTRAL_USERNAME)
50+
- export CENTRAL_PUBLISHER_PASSWORD=$(get_secret $DD_ANDROID_SECRET__PUBLISHING_CENTRAL_PWD)
51+
- export GPG_PUBLIC_FINGERPRINT=$(get_secret $DD_ANDROID_SECRET__SIGNING_GPG_PUBLIC_KEY | gpg --import --import-options show-only | grep -E -o -e "[A-F0-9]{40}")
4852

4953
# CI IMAGE
5054

@@ -183,12 +187,13 @@ test:kover:
183187
- cache/caches/
184188
- cache/notifications/
185189
script:
190+
- !reference [.snippets, source-secrets]
186191
- pip3 install datadog
187192
- rm -rf ~/.gradle/daemon/
188193
- export DD_AGENT_HOST="$BUILDENV_HOST_IP"
189-
- export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.api_key --with-decryption --query "Parameter.Value" --out text)
190-
- export DD_APP_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.app_key --with-decryption --query "Parameter.Value" --out text)
191-
- CODECOV_TOKEN=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.codecov-token --with-decryption --query "Parameter.Value" --out text)
194+
- export DD_API_KEY=$(get_secret $DD_ANDROID_SECRET__API_KEY)
195+
- export DD_APP_KEY=$(get_secret $DD_ANDROID_SECRET__APP_KEY)
196+
- CODECOV_TOKEN=$(get_secret $DD_ANDROID_SECRET__CODECOV_TOKEN)
192197
- GRADLE_OPTS="-Xmx3072m" DD_TAGS="test.configuration.variant:release" ./gradlew :dd-sdk-android-core:koverXmlReportRelease --no-daemon --build-cache --gradle-user-home cache/ -Dorg.gradle.jvmargs=-javaagent:$DD_TRACER_FOLDER/dd-java-agent.jar=$DD_COMMON_AGENT_CONFIG
193198
- GRADLE_OPTS="-Xmx3072m" DD_TAGS="test.configuration.variant:release" ./gradlew :dd-sdk-android-internal:koverXmlReportRelease --no-daemon --build-cache --gradle-user-home cache/ -Dorg.gradle.jvmargs=-javaagent:$DD_TRACER_FOLDER/dd-java-agent.jar=$DD_COMMON_AGENT_CONFIG
194199
- GRADLE_OPTS="-Xmx3072m" DD_TAGS="test.configuration.variant:release" ./gradlew :koverReportFeatures --no-daemon --build-cache --gradle-user-home cache/ -Dorg.gradle.jvmargs=-javaagent:$DD_TRACER_FOLDER/dd-java-agent.jar=$DD_COMMON_AGENT_CONFIG
@@ -373,8 +378,9 @@ test-pyramid:detekt-api-coverage:
373378
stage: test-pyramid
374379
timeout: 1h
375380
script:
381+
- !reference [ .snippets, source-secrets ]
376382
- mkdir -p ./config/
377-
- aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.gradle-properties --with-decryption --query "Parameter.Value" --out text >> ./gradle.properties
383+
- get_secret $DD_ANDROID_SECRET__GRADLE_PROPERTIES > ./gradle.properties
378384
- GRADLE_OPTS="-Xmx4096M" ./gradlew assembleLibrariesDebug --stacktrace --no-daemon
379385
- GRADLE_OPTS="-Xmx4096M" ./gradlew printSdkDebugRuntimeClasspath --stacktrace --no-daemon
380386
- GRADLE_OPTS="-Xmx4096M" ./gradlew :tools:detekt:jar --stacktrace --no-daemon
@@ -391,14 +397,15 @@ test-pyramid:publish-e2e-synthetics:
391397
only:
392398
- develop
393399
script:
400+
- !reference [ .snippets, source-secrets ]
394401
- mkdir -p ./config/
395-
- aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.gradle-properties --with-decryption --query "Parameter.Value" --out text >> ./gradle.properties
396-
- aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.keystore --with-decryption --query "Parameter.Value" --out text | base64 -d > ./sample-android.keystore
397-
- aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.e2e_config_json --with-decryption --query "Parameter.Value" --out text > ./config/us1.json
398-
- export E2E_STORE_PASSWD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.keystore-password --with-decryption --query "Parameter.Value" --out text)
399-
- export E2E_DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.e2e_api_key --with-decryption --query "Parameter.Value" --out text)
400-
- export E2E_DD_APP_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.e2e_app_key --with-decryption --query "Parameter.Value" --out text)
401-
- export E2E_MOBILE_APP_ID=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.e2e_mobile_app_id --with-decryption --query "Parameter.Value" --out text)
402+
- get_secret $DD_ANDROID_SECRET__GRADLE_PROPERTIES > ./gradle.properties
403+
- get_secret $DD_ANDROID_SECRET__KEYSTORE > ./sample-android.keystore
404+
- get_secret $DD_ANDROID_SECRET__E2E_CONFIG_JSON > ./config/us1.json
405+
- export E2E_STORE_PASSWD=$(get_secret $DD_ANDROID_SECRET__KEYSTORE_PWD)
406+
- export E2E_DD_API_KEY=$(get_secret $DD_ANDROID_SECRET__E2E_API_KEY)
407+
- export E2E_DD_APP_KEY=$(get_secret $DD_ANDROID_SECRET__E2E_APP_KEY)
408+
- export E2E_MOBILE_APP_ID=$(get_secret $DD_ANDROID_SECRET__E2E_MOBILE_APP_ID)
402409
- GRADLE_OPTS="-Xmx4096M" ./gradlew assembleLibrariesRelease --stacktrace --no-daemon
403410
- GRADLE_OPTS="-Xmx4096M" ./gradlew :sample:kotlin:packageUs1Release --stacktrace --no-daemon
404411
- npm update -g @datadog/datadog-ci
@@ -418,14 +425,15 @@ test-pyramid:publish-webview-synthetics:
418425
only:
419426
- develop
420427
script:
428+
- !reference [ .snippets, source-secrets ]
421429
- mkdir -p ./config/
422-
- aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.gradle-properties --with-decryption --query "Parameter.Value" --out text >> ./gradle.properties
423-
- aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.keystore --with-decryption --query "Parameter.Value" --out text | base64 -d > ./sample-android.keystore
424-
- aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.webview_config_json --with-decryption --query "Parameter.Value" --out text > ./config/us1.json
425-
- export E2E_STORE_PASSWD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.keystore-password --with-decryption --query "Parameter.Value" --out text)
426-
- export E2E_DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.webview_api_key --with-decryption --query "Parameter.Value" --out text)
427-
- export E2E_DD_APP_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.webview_app_key --with-decryption --query "Parameter.Value" --out text)
428-
- export E2E_MOBILE_APP_ID=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.webview_mobile_app_id --with-decryption --query "Parameter.Value" --out text)
430+
- get_secret $DD_ANDROID_SECRET__GRADLE_PROPERTIES > ./gradle.properties
431+
- get_secret $DD_ANDROID_SECRET__KEYSTORE > ./sample-android.keystore
432+
- get_secret $DD_ANDROID_SECRET__WEBVIEW_CONFIG_JSON > ./config/us1.json
433+
- export E2E_STORE_PASSWD=$(get_secret $DD_ANDROID_SECRET__KEYSTORE_PWD)
434+
- export E2E_DD_API_KEY=$(get_secret $DD_ANDROID_SECRET__WEBVIEW_API_KEY)
435+
- export E2E_DD_APP_KEY=$(get_secret $DD_ANDROID_SECRET__WEBVIEW_APP_KEY)
436+
- export E2E_MOBILE_APP_ID=$(get_secret $DD_ANDROID_SECRET__WEBVIEW_MOBILE_APP_ID)
429437
- GRADLE_OPTS="-Xmx4096M" ./gradlew assembleLibrariesRelease --stacktrace --no-daemon
430438
- GRADLE_OPTS="-Xmx4096M" ./gradlew :sample:kotlin:packageUs1Release --stacktrace --no-daemon
431439
- npm update -g @datadog/datadog-ci
@@ -445,14 +453,15 @@ test-pyramid:publish-staging-synthetics:
445453
only:
446454
- develop
447455
script:
456+
- !reference [ .snippets, source-secrets ]
448457
- mkdir -p ./config/
449-
- aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.gradle-properties --with-decryption --query "Parameter.Value" --out text >> ./gradle.properties
450-
- aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.keystore --with-decryption --query "Parameter.Value" --out text | base64 -d > ./sample-android.keystore
451-
- aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.e2e_staging_config_json --with-decryption --query "Parameter.Value" --out text > ./config/staging.json
452-
- export E2E_STORE_PASSWD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.keystore-password --with-decryption --query "Parameter.Value" --out text)
453-
- export E2E_DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.e2e_staging_api_key --with-decryption --query "Parameter.Value" --out text)
454-
- export E2E_DD_APP_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.e2e_staging_app_key --with-decryption --query "Parameter.Value" --out text)
455-
- export E2E_MOBILE_APP_ID=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.e2e_staging_mobile_app_id --with-decryption --query "Parameter.Value" --out text)
458+
- get_secret $DD_ANDROID_SECRET__GRADLE_PROPERTIES > ./gradle.properties
459+
- get_secret $DD_ANDROID_SECRET__KEYSTORE > ./sample-android.keystore
460+
- get_secret $DD_ANDROID_SECRET__E2E_STAGING_CONFIG_JSON > ./config/staging.json
461+
- export E2E_STORE_PASSWD=$(get_secret $DD_ANDROID_SECRET__KEYSTORE_PWD)
462+
- export E2E_DD_API_KEY=$(get_secret $DD_ANDROID_SECRET__E2E_STAGING_API_KEY)
463+
- export E2E_DD_APP_KEY=$(get_secret $DD_ANDROID_SECRET__E2E_STAGING_APP_KEY)
464+
- export E2E_MOBILE_APP_ID=$(get_secret $DD_ANDROID_SECRET__E2E_STAGING_APP_ID)
456465
- GRADLE_OPTS="-Xmx4096M" ./gradlew assembleLibrariesRelease --stacktrace --no-daemon
457466
- GRADLE_OPTS="-Xmx4096M" ./gradlew :sample:kotlin:packageStagingRelease --stacktrace --no-daemon
458467
- npm update -g @datadog/datadog-ci
@@ -472,14 +481,15 @@ test-pyramid:publish-benchmark-synthetics:
472481
only:
473482
- develop
474483
script:
484+
- !reference [ .snippets, source-secrets ]
475485
- mkdir -p ./config/
476-
- aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.gradle-properties --with-decryption --query "Parameter.Value" --out text >> ./gradle.properties
477-
- aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.keystore --with-decryption --query "Parameter.Value" --out text | base64 -d > ./sample-benchmark.keystore
478-
- aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.benchmark_config_json --with-decryption --query "Parameter.Value" --out text > ./config/benchmark.json
479-
- export BM_STORE_PASSWD=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.keystore-password --with-decryption --query "Parameter.Value" --out text)
480-
- export BM_DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.benchmark_api_key --with-decryption --query "Parameter.Value" --out text)
481-
- export BM_DD_APP_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.benchmark_app_key --with-decryption --query "Parameter.Value" --out text)
482-
- export BM_MOBILE_APP_ID=$(aws ssm get-parameter --region us-east-1 --name ci.dd-sdk-android.benchmark_mobile_app_id --with-decryption --query "Parameter.Value" --out text)
486+
- get_secret $DD_ANDROID_SECRET__GRADLE_PROPERTIES > ./gradle.properties
487+
- get_secret $DD_ANDROID_SECRET__KEYSTORE > ./sample-benchmark.keystore
488+
- get_secret $DD_ANDROID_SECRET__BENCHMARK_CONFIG_JSON > ./config/benchmark.json
489+
- export BM_STORE_PASSWD=$(get_secret $DD_ANDROID_SECRET__KEYSTORE_PWD)
490+
- export BM_DD_API_KEY=$(get_secret $DD_ANDROID_SECRET__BENCHMARK_API_KEY)
491+
- export BM_DD_APP_KEY=$(get_secret $DD_ANDROID_SECRET__BENCHMARK_APP_KEY)
492+
- export BM_MOBILE_APP_ID=$(get_secret $DD_ANDROID_SECRET__BENCHMARK_MOBILE_APP_ID)
483493
- GRADLE_OPTS="-Xmx4096M" ./gradlew assembleLibrariesRelease --stacktrace --no-daemon
484494
- GRADLE_OPTS="-Xmx4096M" ./gradlew :sample:benchmark:packageRelease --stacktrace --no-daemon
485495
- npm update -g @datadog/datadog-ci

‎ci/scripts/get-secret.sh‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/zsh
2+
3+
#
4+
# Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
5+
# This product includes software developed at Datadog (https://www.datadoghq.com/).
6+
# Copyright 2016-Present Datadog, Inc.
7+
#
8+
9+
source ./ci/scripts/vault_config.sh
10+
source ./ci/scripts/list-secrets.sh
11+
12+
# Usage:
13+
# get_secret <secret_name>
14+
#
15+
# Notes:
16+
# - For <secret_name> use constants defined in './ci/scripts/vault_config.sh'
17+
# - Requires `vault` to be installed
18+
get_secret() {
19+
local secret_name=$1
20+
21+
if [ "$CI" = "false" ]; then
22+
# K8s runners don't need to set VAULT_ADDR, they have VAULT_ADDR injected alongside the emissary sidecar container.
23+
export VAULT_ADDR=$DD_VAULT_ADDR
24+
if vault token lookup &>/dev/null; then
25+
echo "Reading '$secret_name' secret in local env. You are already authenticated with 'vault'." >&2
26+
else
27+
echo "Reading '$secret_name' secret in local env. You will now be authenticated with OIDC in your web browser." >&2
28+
vault login -method=oidc -no-print
29+
fi
30+
fi
31+
32+
local secret_value=$(vault kv get -field=value "$DD_ANDROID_SECRETS_PATH_PREFIX/$secret_name")
33+
34+
if [[ -z "$secret_value" ]]; then
35+
echo "Error" "Failed to retrieve the '$secret_name' secret or the secret is empty." >&2
36+
exit 1
37+
fi
38+
39+
echo "$secret_value"
40+
}
41+
42+
# Only run the main logic if the script is executed directly (not sourced)
43+
if [ "$CI" != "true" ]; then
44+
list_secrets
45+
select_secret
46+
get_secret "$SECRET_NAME"
47+
fi

‎ci/scripts/list-secrets.sh‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/bin/zsh
2+
3+
#
4+
# Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
5+
# This product includes software developed at Datadog (https://www.datadoghq.com/).
6+
# Copyright 2016-Present Datadog, Inc.
7+
#
8+
9+
source ./ci/scripts/vault_config.sh
10+
11+
list_secrets() {
12+
GREEN="\e[32m"
13+
RESET="\e[0m"
14+
15+
echo "Available secrets:"
16+
for key in ${(k)DD_ANDROID_SECRETS}; do
17+
IFS=" | " read -r name description <<< "${DD_ANDROID_SECRETS[$key]}"
18+
echo "$key) ${GREEN}$name${RESET} - $description"
19+
done | sort -n
20+
21+
echo ""
22+
echo "To add a new secret, first define it in 'ci/scripts/vault_config.sh' and retry."
23+
}
24+
25+
26+
select_secret() {
27+
echo
28+
while true; do
29+
echo "Enter the number of the secret you want to continue:"
30+
read "secret_number"
31+
if [[ -n ${DD_ANDROID_SECRETS[$secret_number]} ]]; then
32+
IFS=" | " read -r SECRET_NAME SECRET_DESC <<< "${DD_ANDROID_SECRETS[$secret_number]}"
33+
break
34+
else
35+
echo_err "Invalid selection. Please enter a valid number."
36+
fi
37+
done
38+
}

‎ci/scripts/set-secret.sh‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
#!/bin/zsh
2+
3+
#
4+
# Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
5+
# This product includes software developed at Datadog (https://www.datadoghq.com/).
6+
# Copyright 2016-Present Datadog, Inc.
7+
#
8+
9+
# Usage:
10+
# $ ./ci/scripts/vault_config.sh
11+
#
12+
# Note:
13+
# - Requires `vault` to be installed
14+
15+
source ./ci/scripts/vault_config.sh
16+
source ./ci/scripts/list-secrets.sh
17+
18+
select_input_method() {
19+
echo
20+
echo "How would you like to provide the secret value?"
21+
echo "1) Enter manually"
22+
echo "2) Read from text file"
23+
while true; do
24+
echo "Enter your choice:"
25+
read "input_method"
26+
case $input_method in
27+
1)
28+
get_secret_value_from_input
29+
break
30+
;;
31+
2)
32+
get_secret_value_from_file
33+
break
34+
;;
35+
*)
36+
echo "Invalid choice."
37+
;;
38+
esac
39+
done
40+
}
41+
42+
get_secret_value_from_file() {
43+
echo "Enter the file path to read the value for '$SECRET_NAME':"
44+
read "SECRET_FILE"
45+
echo
46+
47+
SECRET_FILE=${SECRET_FILE/#\~/$HOME} # Expand ~ to home directory if present
48+
echo "Using '$SECRET_FILE'"
49+
50+
if [[ -f "$SECRET_FILE" ]]; then
51+
SECRET_VALUE=$(cat "$SECRET_FILE")
52+
else
53+
echo "Error: File '$SECRET_FILE' does not exist."
54+
exit 1
55+
fi
56+
}
57+
58+
get_secret_value_from_input() {
59+
echo "Enter the new value for '$SECRET_NAME':"
60+
read "SECRET_VALUE"
61+
echo
62+
}
63+
64+
set_secret_value() {
65+
echo "You will now be authenticated with OIDC in your web browser. Press ENTER to continue."
66+
read
67+
export VAULT_ADDR=$DD_VAULT_ADDR
68+
vault login -method=oidc -no-print
69+
vault kv put "$DD_ANDROID_SECRETS_PATH_PREFIX/$SECRET_NAME" value="$SECRET_VALUE"
70+
echo "Secret '$SECRET_NAME' set successfully."
71+
}
72+
73+
list_secrets
74+
select_secret
75+
select_input_method
76+
set_secret_value

0 commit comments

Comments
 (0)