[8.19](backport #4895) bk: use GCP OIDC (#4932)

Author: mergify[bot]

.buildkite/hooks/pre-command

@@ -6,11 +6,12 @@ source .buildkite/scripts/common.sh
 
 DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
 EC_KEY_SECRET_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
-PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
 CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
-JOB_GCS_BUCKET="ingest-buildkite-ci"
+JOB_GCS_BUCKET="fleet-server-ci-internal"
 GITHUB_REPO_TOKEN=$VAULT_GITHUB_TOKEN
 
+export JOB_GCS_BUCKET
+
 # Usage:
 #check_if_file_exist_in_repo "infra" "main"
 #Returns FILE_EXISTS_IN_REPO=true if the defined file exists in the difined repo and FILE_EXISTS_IN_REPO=false if not exists
@@ -67,18 +68,6 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" ]]; then
   fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "release-test" ]]; then
-  export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
-  export JOB_GCS_BUCKET
-fi
-
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == package-x86-64* || "$BUILDKITE_STEP_KEY" == package-fips-x86_64* || "$BUILDKITE_STEP_KEY" == package-arm* || "$BUILDKITE_STEP_KEY" == package-fips-arm* || "$BUILDKITE_STEP_KEY" == "dra-snapshot" || "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
-    export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
-    export JOB_GCS_BUCKET
-  fi
-fi
-
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "dra-snapshot" || "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
     export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
@@ -90,10 +79,3 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
     export VAULT_SECRET_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.secret_id')
   fi
 fi
-
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == package-x86-64* || "$BUILDKITE_STEP_KEY" == package-fips-x86-64* || "$BUILDKITE_STEP_KEY" == package-arm* || "$BUILDKITE_STEP_KEY" == package-fips-arm* ]]; then
-    export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
-    export JOB_GCS_BUCKET
-  fi
-fi

.buildkite/hooks/pre-exit

@@ -11,13 +11,11 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" || "$BUILDKITE_PIPELINE_SLUG"
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "release-test" ]]; then
-    unset GOOGLE_APPLICATION_CREDENTIALS
     cleanup
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == package-x86-64* || "$BUILDKITE_STEP_KEY" == package-fips-x86-64* || "$BUILDKITE_STEP_KEY" == package-arm* || "$BUILDKITE_STEP_KEY" == package-fips-arm* || "$BUILDKITE_STEP_KEY" == "dra-snapshot" && "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
-    unset GOOGLE_APPLICATION_CREDENTIALS
     unset VAULT_ROLE_ID_SECRET
     unset VAULT_ADDR_SECRET
     unset VAULT_SECRET_ID_SECRET

.buildkite/pipeline.package.mbp.yml

@@ -6,6 +6,17 @@ env:
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2004"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
 
+# This section is used to define the plugins that will be used in the pipeline.
+# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
+common:
+  - oidc_plugin: &oidc_plugin
+      # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/fleet-server/01-gcp-buildkite-oidc.tf
+      # This plugin authenticates to Google Cloud using the OIDC token.
+      elastic/oblt-google-auth#v1.2.0:
+        lifetime: 10800 # seconds
+        project-id: "elastic-observability-ci"
+        project-number: "911195782929"
+
 steps:
   - label: "Package x86_64 snapshot"
     # skip building + packaging snapshot for pre-releases (flagged by a non-empty VERSION_QUALIFIER env var/BK param)
@@ -17,6 +28,8 @@ steps:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
       machineType: "c2-standard-16"
+    plugins:
+      - *oidc_plugin
 
   - label: "Package x86_64 staging"
     key: "package-x86-64-staging"
@@ -27,6 +40,8 @@ steps:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
       machineType: "c2-standard-16"
+    plugins:
+      - *oidc_plugin
 
   - label: "Package FIPS x86_64 snapshot"
     if: "build.env('VERSION_QUALIFIER') == null"
@@ -38,6 +53,8 @@ steps:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
       machineType: "c2-standard-16"
+    plugins:
+      - *oidc_plugin
 
   - label: "Package FIPS x86_64 staging"
     key: "package-fips-x86-64-staging"
@@ -50,6 +67,8 @@ steps:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
       machineType: "c2-standard-16"
+    plugins:
+      - *oidc_plugin
 
   - label: "Package aarch64 snapshot"
     if: "build.env('VERSION_QUALIFIER') == null"
@@ -59,6 +78,8 @@ steps:
       provider: "aws"
       imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
       instanceType: "t4g.2xlarge"
+    plugins:
+      - *oidc_plugin
 
   - label: "Package aarch64 staging"
     key: "package-arm-staging"
@@ -69,6 +90,8 @@ steps:
       provider: "aws"
       imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
       instanceType: "t4g.2xlarge"
+    plugins:
+      - *oidc_plugin
 
   - label: "Package FIPS aarch64 snapshot"
     if: "build.env('VERSION_QUALIFIER') == null"
@@ -80,6 +103,8 @@ steps:
       provider: "aws"
       imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
       instanceType: "t4g.2xlarge"
+    plugins:
+      - *oidc_plugin
 
   - label: "Package FIPS aarch64 staging"
     key: "package-fips-arm-staging"
@@ -92,6 +117,8 @@ steps:
       provider: "aws"
       imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
       instanceType: "t4g.2xlarge"
+    plugins:
+      - *oidc_plugin
 
   - label: "DRA snapshot"
     if: "${FILE_EXISTS_IN_REPO} && build.env('VERSION_QUALIFIER') == null"
@@ -101,6 +128,8 @@ steps:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
       machineType: "c2-standard-16"
+    plugins:
+      - *oidc_plugin
     depends_on:
       - step: "package-x86-64-snapshot"
         allow_failure: false
@@ -123,6 +152,8 @@ steps:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
       machineType: "c2-standard-16"
+    plugins:
+      - *oidc_plugin
     depends_on:
       - step: "package-x86-64-staging"
         allow_failure: false

.buildkite/pipeline.yml

@@ -230,6 +230,13 @@ steps:
     depends_on:
       - step: "tests"
         allow_failure: false
+    plugins:
+      # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/fleet-server/01-gcp-buildkite-oidc.tf
+      # This plugin authenticates to Google Cloud using the OIDC token.
+      - elastic/oblt-google-auth#v1.2.0:
+          lifetime: 10800 # seconds
+          project-id: "elastic-observability-ci"
+          project-number: "911195782929"
 
   - label: ":jenkins: Release - Package Registry Distribution"
     key: "release-package-registry"

.buildkite/scripts/common.sh

@@ -107,13 +107,6 @@ with_Terraform() {
     terraform version
 }
 
-google_cloud_auth() {
-    local secretFileLocation=$(mktemp -d -p "${WORKSPACE}" -t "${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX")/google-cloud-credentials.json
-    echo "${PRIVATE_CI_GCS_CREDENTIALS_SECRET}" > ${secretFileLocation}
-    gcloud auth activate-service-account --key-file ${secretFileLocation} 2> /dev/null
-    export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
-}
-
 upload_packages_to_gcp_bucket() {
     local pattern=${1}
     local baseUri="gs://${JOB_GCS_BUCKET}/${REPO}"
@@ -124,7 +117,7 @@ upload_packages_to_gcp_bucket() {
         bucketUriDefault="${baseUri}/pull-requests/pr-${GITHUB_PR_NUMBER}"
     fi
     for bucketUri in "${bucketUriCommit}" "${bucketUriDefault}"; do
-        gsutil -m -q cp -r ${pattern} "${bucketUri}"
+        gcloud storage cp --recursive --quiet ${pattern} "${bucketUri}"
     done
 }
 
@@ -143,15 +136,15 @@ upload_mbp_packages_to_gcp_bucket() {
     local pattern=${1}
     local type=${2}
     get_bucket_uri "${type}"
-    gsutil -m -q cp -r ${pattern} ${bucketUri}
+    gcloud storage cp --recursive --quiet ${pattern} ${bucketUri}
 }
 
 download_mbp_packages_from_gcp_bucket() {
     local pattern=${1}
     local type=${2}
     mkdir -p ${WORKSPACE}/${pattern}
     get_bucket_uri "${type}"
-    gsutil -m -q cp -r ${bucketUri}/* ${WORKSPACE}/${pattern}
+    gcloud storage cp --recursive --quiet ${bucketUri}/* ${WORKSPACE}/${pattern}
 }
 
 with_mage() {

.buildkite/scripts/dra_release.sh

@@ -26,8 +26,6 @@ fi
 
 add_bin_path
 
-google_cloud_auth
-
 download_mbp_packages_from_gcp_bucket "${FOLDER_PATH}" "${TYPE}"
 
 with_go

.buildkite/scripts/package.sh

@@ -45,5 +45,4 @@ case "${TYPE}" in
     ;;
 esac
 
-google_cloud_auth
 upload_mbp_packages_to_gcp_bucket "build/distributions/**/*" "${TYPE}"

.buildkite/scripts/release_test.sh

@@ -4,9 +4,9 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
-echo "Checking gsutil command..."
-if ! command -v gsutil &> /dev/null ; then
-    echo "⚠️ gsutil is not installed"
+echo "Checking gcloud command..."
+if ! command -v gcloud &> /dev/null ; then
+    echo "⚠️ gcloud is not installed"
     exit 1
 fi
 
@@ -16,8 +16,6 @@ with_go
 
 make docker-release
 
-google_cloud_auth
-
 upload_packages_to_gcp_bucket "build/distributions/"
 
 make test-release