[8.19](backport #5042) bk: use buildkite plugins (#5103)

Author: mergify[bot]

.buildkite/hooks/pre-command

@@ -4,7 +4,6 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
-DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
 EC_KEY_SECRET_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
 CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
 JOB_GCS_BUCKET="fleet-server-ci-internal"
@@ -45,14 +44,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
   check_if_file_exist_in_repo "infra" "${_branch}"                  #TODO should be changed to "main" for rollback...
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" || "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-perf-tests" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "publish" || "$BUILDKITE_STEP_KEY" == "cloud-e2e-test" || "$BUILDKITE_STEP_KEY" == "cloud-e2e-fips-test"  || "$BUILDKITE_STEP_KEY" == "create-image" ]]; then
-    export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
-    export DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
-    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
-  fi
-fi
-
+# TODO: use a builkite plugin to handle this
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "cloud-e2e-test" || "$BUILDKITE_STEP_KEY" == "cloud-e2e-fips-test"  ]]; then
     export EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey "${EC_KEY_SECRET_PATH}")
@@ -61,20 +53,9 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" ]]; then
   fi
 fi
 
-# BK analytics
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "int-test" || "$BUILDKITE_STEP_KEY" == "e2e-test" || "$BUILDKITE_STEP_KEY" == "fips-e2e-test" ]]; then
-    echo "--- Prepare BK test analytics token :vault:"
-    BUILDKITE_ANALYTICS_TOKEN=$(vault kv get -field token kv/ci-shared/platform-ingest/buildkite_fleet_server_analytics_token)
-    export BUILDKITE_ANALYTICS_TOKEN
-  fi
-fi
-
+# TODO: use a builkite plugin to handle this
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "dra-snapshot" || "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
-    export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
-    export DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
-    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
     DRA_CREDS_SECRET=$(retry 5 vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH})
     export VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr')
     export VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id')

.buildkite/hooks/pre-exit

@@ -4,12 +4,6 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" || "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-perf-tests" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "publish" || "$BUILDKITE_STEP_KEY" == "cloud-e2e-test" || "$BUILDKITE_STEP_KEY" == "create-image" ]]; then
-    docker logout ${DOCKER_REGISTRY}
-  fi
-fi
-
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "release-test" ]]; then
     cleanup
 fi
@@ -19,7 +13,6 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
     unset VAULT_ROLE_ID_SECRET
     unset VAULT_ADDR_SECRET
     unset VAULT_SECRET_ID_SECRET
-    docker logout ${DOCKER_REGISTRY}
     cleanup
   fi
 fi

.buildkite/pipeline.package.mbp.yml

@@ -2,7 +2,6 @@
 name: "fleet server package mbp"
 env:
   REPO: 'fleet-server'
-  DOCKER_REGISTRY: "docker.elastic.co"
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2004"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
 

.buildkite/pipeline.perf-tests.yaml

@@ -1,18 +1,26 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 env:
-  DOCKER_REGISTRY: "docker.elastic.co"
-  DOCKER_IMAGE: "${DOCKER_REGISTRY}/observability-ci/fleet-server" # needs to rename for rollback
+  DOCKER_IMAGE: "docker.elastic.co/observability-ci/fleet-server" # needs to rename for rollback
   DOCKER_IMAGE_GIT_TAG: "${BUILDKITE_BRANCH}" # needs to rename for rollback
   DOCKER_IMAGE_LATEST_TAG: "latest" # needs to rename for rollback
   DOCKER_IMAGE_SHA_TAG: "git-${BUILDKITE_COMMIT:0:12}" # needs to rename for rollback, should be "git-${BUILDKITE_COMMIT:0:12}"
 
+# This section is used to define the plugins that will be used in the pipeline.
+# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
+common:
+  - docker_elastic_login_plugin: &docker_elastic_login_plugin
+      elastic/vault-docker-login#v0.6.0:
+        secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry'
+
 steps:
   - label: ":docker: Publish docker image"
     key: "create-image"
     command: ".buildkite/scripts/build_push_docker_image.sh"
     agents:
       provider: "gcp"
+    plugins:
+      - *docker_elastic_login_plugin
 
   - label: "perf test"
     key: "obs-perf-test"

.buildkite/pipeline.yml

@@ -5,6 +5,24 @@ env:
   TERRAFORM_VERSION: "1.6.4"
   IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-fleet-server-ubuntu-2204-fips-1751684469"
 
+# This section is used to define the plugins that will be used in the pipeline.
+# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
+common:
+  - test_collector_plugin: &test_collector_plugin
+      test-collector#v1.11.0:
+        files: "build/test-*.xml"
+        format: "junit"
+        branches: "main"
+        debug: true
+  - bk_analytics_token_plugin: &bk_analytics_token_plugin
+      elastic/vault-secrets#v0.1.0:
+        path: "kv/ci-shared/platform-ingest/buildkite_analytics_token"
+        field: "token"
+        env_var: "BUILDKITE_ANALYTICS_TOKEN"
+  - docker_elastic_login_plugin: &docker_elastic_login_plugin
+      elastic/vault-docker-login#v0.6.0:
+        secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry'
+
 steps:
   - group: "Check and build"
     key: "check"
@@ -174,7 +192,6 @@ steps:
       - label: ":gcloud: Cloud e2e Test"
         key: "cloud-e2e-test"
         env:
-          DOCKER_REGISTRY: "docker.elastic.co"
           DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fleet"
           DOCKER_IMAGE_TAG: "pr-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_COMMIT:0:12}"
           SNAPSHOT: "true"
@@ -183,6 +200,8 @@ steps:
         command: ".buildkite/scripts/cloud_e2e_test.sh"
         agents:
           provider: "gcp"
+        plugins:
+          - *docker_elastic_login_plugin
         depends_on:
           - step: "unit-test"
             allow_failure: false
@@ -199,7 +218,6 @@ steps:
       - label: ":gcloud: Cloud e2e FIPS Test"
         key: "cloud-e2e-fips-test"
         env:
-          DOCKER_REGISTRY: "docker.elastic.co"
           DOCKER_BASE_IMAGE: "docker.elastic.co/cloud-release/elastic-agent-cloud-fips"
           DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips"
           DOCKER_IMAGE_TAG: "pr-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_COMMIT:0:12}"
@@ -210,6 +228,8 @@ steps:
         command: ".buildkite/scripts/cloud_e2e_test.sh"
         agents:
           provider: "gcp"
+        plugins:
+          - *docker_elastic_login_plugin
         depends_on:
           - step: "unit-test"
             allow_failure: false
@@ -227,14 +247,15 @@ steps:
     key: "publish"
     command: ".buildkite/scripts/build_push_docker_image.sh"
     env:
-      DOCKER_REGISTRY: "docker.elastic.co"
       DOCKER_IMAGE: "docker.elastic.co/observability-ci/fleet-server" # needs to rename for rollback
       DOCKER_IMAGE_SHA_TAG: "git-${BUILDKITE_COMMIT:0:12}" # needs to rename for rollback, should be "git-${BUILDKITE_COMMIT:0:12}"
       DOCKER_IMAGE_LATEST_TAG: "latest" # needs to rename for rollback
       DOCKER_IMAGE_GIT_TAG: "${BUILDKITE_BRANCH}" # needs to rename for rollback
     if: "build.env('BUILDKITE_PULL_REQUEST') == 'false' && build.env('BUILDKITE_BRANCH') == 'main'"
     agents:
       provider: "gcp"
+    plugins:
+      - *docker_elastic_login_plugin
     depends_on:
       - step: "tests"
         allow_failure: false

.buildkite/scripts/common.sh

@@ -93,11 +93,6 @@ retry() {
     return 0
 }
 
-docker_logout() {
-    echo "Logging out from Docker..."
-    docker logout ${DOCKER_REGISTRY}
-}
-
 with_Terraform() {
     echo "Setting up the Terraform environment..."
     local path_to_file="${WORKSPACE}/terraform.zip"