wip

Author: ambushwork

ci/pipelines/default-pipeline.yml

@@ -4,7 +4,6 @@ include:
 # SETUP
 
 stages:
-  - fetch-secrets
   - ci-image
   - security
   - analysis
@@ -14,9 +13,9 @@ stages:
   - notify
 
 .snippets:
-  fetch-secrets:
-    - mkdir -p ./ci/pipelines/secrets
-    - ./ci/scripts/fetch-secrets.sh
+  source-secrets:
+    - source ./ci/scripts/vault_config.sh
+    - source ./ci/scripts/get-secret.sh
 
   # macOS AMI will already have cmdline-tools installed
   install-android-api-components:
@@ -44,27 +43,15 @@ stages:
     - if [[ "$exit_code" -ne 0 ]]; then exit 1; fi
     - exit 0
   set-publishing-credentials:
-    - cp ./ci/pipelines/secrets/gradle.properties ./gradle.properties
-    - export GPG_PRIVATE_KEY=$(cat ./ci/pipelines/secrets/gpg_private_key)
-    - export GPG_PASSWORD=$(cat ./ci/pipelines/secrets/gpg_passphrase)
-    - export CENTRAL_PUBLISHER_USERNAME=$(cat ./ci/pipelines/secrets/central_username)
-    - export CENTRAL_PUBLISHER_PASSWORD=$(cat ./ci/pipelines/secrets/central_password)
-    - export GPG_PUBLIC_FINGERPRINT=$(cat ./ci/pipelines/secrets/gpg_public_key | gpg --import --import-options show-only | grep -E -o -e "[A-F0-9]{40}")
+    - get_secret $DD_ANDROID_SECRET__GRADLE_PROPERTIES > ./gradle.properties
+    - export GPG_PRIVATE_KEY=$(get_secret $DD_ANDROID_SECRET__SIGNING_GPG_PRIVATE_KEY)
+    - export GPG_PASSWORD=$(get_secret $DD_ANDROID_SECRET__SIGNING_GPG_PASSPHRASE)
+    - export CENTRAL_PUBLISHER_USERNAME=$(get_secret $DD_ANDROID_SECRET__PUBLISHING_CENTRAL_USERNAME)
+    - export CENTRAL_PUBLISHER_PASSWORD=$(get_secret $DD_ANDROID_SECRET__PUBLISHING_CENTRAL_PWD)
+    - export GPG_PUBLIC_FINGERPRINT=$(get_secret $DD_ANDROID_SECRET__SIGNING_GPG_PUBLIC_KEY | gpg --import --import-options show-only | grep -E -o -e "[A-F0-9]{40}")
 
 # CI IMAGE
 
-fetch-secrets:
-  stage: fetch-secrets
-  tags: ["macos:sonoma","specific:true"]
-  image: $CI_IMAGE_DOCKER
-  script:
-    - !reference [.snippets, fetch-secrets]
-  artifacts:
-    paths:
-      - ./ci/pipelines/secrets/
-    expire_in: 1 hour
-    when: always
-
 ci-image:
   stage: ci-image
   when: manual
@@ -199,12 +186,14 @@ test:kover:
       - cache/caches/
       - cache/notifications/
   script:
+    - !reference [.snippets, source-secrets]
     - pip3 install datadog
     - rm -rf ~/.gradle/daemon/
     - export DD_AGENT_HOST="$BUILDENV_HOST_IP"
-    - export DD_API_KEY=$(cat ./ci/pipelines/secrets/api_key)
-    - export DD_APP_KEY=$(cat ./ci/pipelines/secrets/app_key)
-    - CODECOV_TOKEN=$(cat ./ci/pipelines/secrets/codecov_token)
+    - export DD_API_KEY=$(get_secret $DD_ANDROID_SECRET__API_KEY)
+    - vault token lookup
+    - export DD_APP_KEY=$(get_secret $DD_ANDROID_SECRET__APP_KEY)
+    - CODECOV_TOKEN=$(get_secret $DD_ANDROID_SECRET__CODECOV_TOKEN)
     - GRADLE_OPTS="-Xmx3072m" DD_TAGS="test.configuration.variant:release" ./gradlew :dd-sdk-android-core:koverXmlReportRelease  --no-daemon --build-cache --gradle-user-home cache/ -Dorg.gradle.jvmargs=-javaagent:$DD_TRACER_FOLDER/dd-java-agent.jar=$DD_COMMON_AGENT_CONFIG
     - GRADLE_OPTS="-Xmx3072m" DD_TAGS="test.configuration.variant:release" ./gradlew :dd-sdk-android-internal:koverXmlReportRelease  --no-daemon --build-cache --gradle-user-home cache/ -Dorg.gradle.jvmargs=-javaagent:$DD_TRACER_FOLDER/dd-java-agent.jar=$DD_COMMON_AGENT_CONFIG
     - GRADLE_OPTS="-Xmx3072m" DD_TAGS="test.configuration.variant:release" ./gradlew :koverReportFeatures  --no-daemon --build-cache --gradle-user-home cache/ -Dorg.gradle.jvmargs=-javaagent:$DD_TRACER_FOLDER/dd-java-agent.jar=$DD_COMMON_AGENT_CONFIG
@@ -389,8 +378,9 @@ test-pyramid:detekt-api-coverage:
   stage: test-pyramid
   timeout: 1h
   script:
+    - !reference [ .snippets, source-secrets ]
     - mkdir -p ./config/
-    - cp ./ci/pipelines/secrets/gradle.properties ./gradle.properties
+    - get_secret $DD_ANDROID_SECRET__GRADLE_PROPERTIES > ./gradle.properties
     - GRADLE_OPTS="-Xmx4096M" ./gradlew assembleLibrariesDebug --stacktrace --no-daemon
     - GRADLE_OPTS="-Xmx4096M" ./gradlew printSdkDebugRuntimeClasspath --stacktrace --no-daemon
     - GRADLE_OPTS="-Xmx4096M" ./gradlew :tools:detekt:jar --stacktrace --no-daemon
@@ -407,14 +397,15 @@ test-pyramid:publish-e2e-synthetics:
   only:
     - develop
   script:
+    - !reference [ .snippets, source-secrets ]
     - mkdir -p ./config/
-    - cp ./ci/pipelines/secrets/gradle.properties ./gradle.properties
-    - cp ./ci/pipelines/secrets/keystore ./sample-android.keystore
-    - cp ./ci/pipelines/secrets/e2e_config.json ./config/us1.json
-    - export E2E_STORE_PASSWD=$(cat ./ci/pipelines/secrets/keystore_password)
-    - export E2E_DD_API_KEY=$(cat ./ci/pipelines/secrets/e2e_api_key)
-    - export E2E_DD_APP_KEY=$(cat ./ci/pipelines/secrets/e2e_app_key)
-    - export E2E_MOBILE_APP_ID=$(cat ./ci/pipelines/secrets/e2e_mobile_app_id)
+    - get_secret $DD_ANDROID_SECRET__GRADLE_PROPERTIES > ./gradle.properties
+    - get_secret $DD_ANDROID_SECRET__KEYSTORE > ./sample-android.keystore
+    - get_secret $DD_ANDROID_SECRET__E2E_CONFIG_JSON > ./config/us1.json
+    - export E2E_STORE_PASSWD=$(get_secret $DD_ANDROID_SECRET__KEYSTORE_PWD)
+    - export E2E_DD_API_KEY=$(get_secret $DD_ANDROID_SECRET__E2E_API_KEY)
+    - export E2E_DD_APP_KEY=$(get_secret $DD_ANDROID_SECRET__E2E_APP_KEY)
+    - export E2E_MOBILE_APP_ID=$(get_secret $DD_ANDROID_SECRET__E2E_MOBILE_APP_ID)
     - GRADLE_OPTS="-Xmx4096M" ./gradlew assembleLibrariesRelease --stacktrace --no-daemon
     - GRADLE_OPTS="-Xmx4096M" ./gradlew :sample:kotlin:packageUs1Release --stacktrace --no-daemon
     - npm update -g @datadog/datadog-ci
@@ -434,14 +425,15 @@ test-pyramid:publish-webview-synthetics:
   only:
     - develop
   script:
+    - !reference [ .snippets, source-secrets ]
     - mkdir -p ./config/
-    - cp ./ci/pipelines/secrets/gradle.properties ./gradle.properties
-    - cp ./ci/pipelines/secrets/keystore ./sample-android.keystore
-    - cp ./ci/pipelines/secrets/webview_config.json ./config/us1.json
-    - export E2E_STORE_PASSWD=$(cat ./ci/pipelines/secrets/keystore_password)
-    - export E2E_DD_API_KEY=$(cat ./ci/pipelines/secrets/webview_api_key)
-    - export E2E_DD_APP_KEY=$(cat ./ci/pipelines/secrets/webview_app_key)
-    - export E2E_MOBILE_APP_ID=$(cat ./ci/pipelines/secrets/webview_mobile_app_id)
+    - get_secret $DD_ANDROID_SECRET__GRADLE_PROPERTIES > ./gradle.properties
+    - get_secret $DD_ANDROID_SECRET__KEYSTORE > ./sample-android.keystore
+    - get_secret $DD_ANDROID_SECRET__WEBVIEW_CONFIG_JSON > ./config/us1.json
+    - export E2E_STORE_PASSWD=$(get_secret $DD_ANDROID_SECRET__KEYSTORE_PWD)
+    - export E2E_DD_API_KEY=$(get_secret $DD_ANDROID_SECRET__WEBVIEW_API_KEY)
+    - export E2E_DD_APP_KEY=$(get_secret $DD_ANDROID_SECRET__WEBVIEW_APP_KEY)
+    - export E2E_MOBILE_APP_ID=$(get_secret $DD_ANDROID_SECRET__WEBVIEW_MOBILE_APP_ID)
     - GRADLE_OPTS="-Xmx4096M" ./gradlew assembleLibrariesRelease --stacktrace --no-daemon
     - GRADLE_OPTS="-Xmx4096M" ./gradlew :sample:kotlin:packageUs1Release --stacktrace --no-daemon
     - npm update -g @datadog/datadog-ci
@@ -461,14 +453,15 @@ test-pyramid:publish-staging-synthetics:
   only:
     - develop
   script:
+    - !reference [ .snippets, source-secrets ]
     - mkdir -p ./config/
-    - cp ./ci/pipelines/secrets/gradle.properties ./gradle.properties
-    - cp ./ci/pipelines/secrets/keystore ./sample-android.keystore
-    - cp ./ci/pipelines/secrets/e2e_staging_config.json ./config/staging.json
-    - export E2E_STORE_PASSWD=$(cat ./ci/pipelines/secrets/keystore_password)
-    - export E2E_DD_API_KEY=$(cat ./ci/pipelines/secrets/e2e_staging_api_key)
-    - export E2E_DD_APP_KEY=$(cat ./ci/pipelines/secrets/e2e_staging_app_key)
-    - export E2E_MOBILE_APP_ID=$(cat ./ci/pipelines/secrets/e2e_staging_app_id)
+    - get_secret $DD_ANDROID_SECRET__GRADLE_PROPERTIES > ./gradle.properties
+    - get_secret $DD_ANDROID_SECRET__KEYSTORE > ./sample-android.keystore
+    - get_secret $DD_ANDROID_SECRET__E2E_STAGING_CONFIG_JSON > ./config/staging.json
+    - export E2E_STORE_PASSWD=$(get_secret $DD_ANDROID_SECRET__KEYSTORE_PWD)
+    - export E2E_DD_API_KEY=$(get_secret $DD_ANDROID_SECRET__E2E_STAGING_API_KEY)
+    - export E2E_DD_APP_KEY=$(get_secret $DD_ANDROID_SECRET__E2E_STAGING_APP_KEY)
+    - export E2E_MOBILE_APP_ID=$(get_secret $DD_ANDROID_SECRET__E2E_STAGING_APP_ID)
     - GRADLE_OPTS="-Xmx4096M" ./gradlew assembleLibrariesRelease --stacktrace --no-daemon
     - GRADLE_OPTS="-Xmx4096M" ./gradlew :sample:kotlin:packageStagingRelease --stacktrace --no-daemon
     - npm update -g @datadog/datadog-ci
@@ -488,14 +481,15 @@ test-pyramid:publish-benchmark-synthetics:
   only:
     - develop
   script:
+    - !reference [ .snippets, source-secrets ]
     - mkdir -p ./config/
-    - cp ./ci/pipelines/secrets/gradle.properties ./gradle.properties
-    - cp ./ci/pipelines/secrets/keystore ./sample-benchmark.keystore
-    - cp ./ci/pipelines/secrets/benchmark_config.json ./config/benchmark.json
-    - export BM_STORE_PASSWD=$(cat ./ci/pipelines/secrets/keystore_password)
-    - export BM_DD_API_KEY=$(cat ./ci/pipelines/secrets/benchmark_api_key)
-    - export BM_DD_APP_KEY=$(cat ./ci/pipelines/secrets/benchmark_app_key)
-    - export BM_MOBILE_APP_ID=$(cat ./ci/pipelines/secrets/benchmark_mobile_app_id)
+    - get_secret $DD_ANDROID_SECRET__GRADLE_PROPERTIES > ./gradle.properties
+    - get_secret $DD_ANDROID_SECRET__KEYSTORE >  ./sample-benchmark.keystore
+    - get_secret $DD_ANDROID_SECRET__BENCHMARK_CONFIG_JSON > ./config/benchmark.json
+    - export BM_STORE_PASSWD=$(get_secret $DD_ANDROID_SECRET__KEYSTORE_PWD)
+    - export BM_DD_API_KEY=$(get_secret $DD_ANDROID_SECRET__BENCHMARK_API_KEY)
+    - export BM_DD_APP_KEY=$(get_secret $DD_ANDROID_SECRET__BENCHMARK_APP_KEY)
+    - export BM_MOBILE_APP_ID=$(get_secret $DD_ANDROID_SECRET__BENCHMARK_MOBILE_APP_ID)
     - GRADLE_OPTS="-Xmx4096M" ./gradlew assembleLibrariesRelease --stacktrace --no-daemon
     - GRADLE_OPTS="-Xmx4096M" ./gradlew :sample:benchmark:packageRelease --stacktrace --no-daemon
     - npm update -g @datadog/datadog-ci

ci/scripts/get-secret.sh

@@ -18,12 +18,9 @@ source ./ci/scripts/list-secrets.sh
 get_secret() {
     local secret_name=$1
 
-    export VAULT_ADDR=$DD_VAULT_ADDR
-
-    if [ "$CI" = "true" ]; then
-        echo "Login as CI"
-        vault login -method=aws -no-print
-    else
+    if [ "$CI" = "false" ]; then
+        # K8s runners don't need to set VAULT_ADDR, they have VAULT_ADDR injected alongside the emissary sidecar container.
+        export VAULT_ADDR=$DD_VAULT_ADDR
         if vault token lookup &>/dev/null; then
             echo "Reading '$secret_name' secret in local env. You are already authenticated with 'vault'." >&2
         else

ci/scripts/vault_config.sh

@@ -7,7 +7,7 @@
 #
 
 DD_VAULT_ADDR=https://vault.us1.ddbuild.io
-DD_ANDROID_SECRETS_PATH_PREFIX='kv/aws/arn:aws:iam::486234852809:role/ci-dd-sdk-android/'
+DD_ANDROID_SECRETS_PATH_PREFIX='kv/k8s/gitlab-runner/dd-sdk-android/'
 
 DD_ANDROID_SECRET__TEST_SECRET="test.secret"
 DD_ANDROID_SECRET__GRADLE_PROPERTIES="gradle.properties"