Merge branch '8.19' into mergify/bp/8.19/pr-5123

Author: v1v

.buildkite/hooks/pre-command

@@ -4,7 +4,6 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
-DOCKER_REGISTRY_SECRET_PATH="kv/ci-shared/platform-ingest/docker_registry_prod"
 EC_KEY_SECRET_PATH="kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
 CI_DRA_ROLE_PATH="kv/ci-shared/release/dra-role"
 JOB_GCS_BUCKET="fleet-server-ci-internal"
@@ -45,14 +44,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
   check_if_file_exist_in_repo "infra" "${_branch}"                  #TODO should be changed to "main" for rollback...
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" || "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-perf-tests" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "publish" || "$BUILDKITE_STEP_KEY" == "cloud-e2e-test" || "$BUILDKITE_STEP_KEY" == "cloud-e2e-fips-test"  || "$BUILDKITE_STEP_KEY" == "create-image" ]]; then
-    export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
-    export DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
-    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
-  fi
-fi
-
+# TODO: use a builkite plugin to handle this
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "cloud-e2e-test" || "$BUILDKITE_STEP_KEY" == "cloud-e2e-fips-test"  ]]; then
     export EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey "${EC_KEY_SECRET_PATH}")
@@ -61,20 +53,9 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" ]]; then
   fi
 fi
 
-# BK analytics
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "int-test" || "$BUILDKITE_STEP_KEY" == "e2e-test" || "$BUILDKITE_STEP_KEY" == "fips-e2e-test" ]]; then
-    echo "--- Prepare BK test analytics token :vault:"
-    BUILDKITE_ANALYTICS_TOKEN=$(vault kv get -field token kv/ci-shared/platform-ingest/buildkite_fleet_server_analytics_token)
-    export BUILDKITE_ANALYTICS_TOKEN
-  fi
-fi
-
+# TODO: use a builkite plugin to handle this
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "dra-snapshot" || "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
-    export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
-    export DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
-    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
     DRA_CREDS_SECRET=$(retry 5 vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH})
     export VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr')
     export VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id')

.buildkite/hooks/pre-exit

@@ -4,12 +4,6 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" || "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-perf-tests" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "publish" || "$BUILDKITE_STEP_KEY" == "cloud-e2e-test" || "$BUILDKITE_STEP_KEY" == "create-image" ]]; then
-    docker logout ${DOCKER_REGISTRY}
-  fi
-fi
-
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "release-test" ]]; then
     cleanup
 fi
@@ -19,7 +13,6 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
     unset VAULT_ROLE_ID_SECRET
     unset VAULT_ADDR_SECRET
     unset VAULT_SECRET_ID_SECRET
-    docker logout ${DOCKER_REGISTRY}
     cleanup
   fi
 fi

.buildkite/pipeline.package.mbp.yml

@@ -2,9 +2,8 @@
 name: "fleet server package mbp"
 env:
   REPO: 'fleet-server'
-  DOCKER_REGISTRY: "docker.elastic.co"
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
-  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2204-aarch64"
+  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2004-aarch64"
 
 # This section is used to define the plugins that will be used in the pipeline.
 # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins

.buildkite/pipeline.perf-tests.yaml

@@ -1,18 +1,26 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
 
 env:
-  DOCKER_REGISTRY: "docker.elastic.co"
-  DOCKER_IMAGE: "${DOCKER_REGISTRY}/observability-ci/fleet-server" # needs to rename for rollback
+  DOCKER_IMAGE: "docker.elastic.co/observability-ci/fleet-server" # needs to rename for rollback
   DOCKER_IMAGE_GIT_TAG: "${BUILDKITE_BRANCH}" # needs to rename for rollback
   DOCKER_IMAGE_LATEST_TAG: "latest" # needs to rename for rollback
   DOCKER_IMAGE_SHA_TAG: "git-${BUILDKITE_COMMIT:0:12}" # needs to rename for rollback, should be "git-${BUILDKITE_COMMIT:0:12}"
 
+# This section is used to define the plugins that will be used in the pipeline.
+# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
+common:
+  - docker_elastic_login_plugin: &docker_elastic_login_plugin
+      elastic/vault-docker-login#v0.6.0:
+        secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry'
+
 steps:
   - label: ":docker: Publish docker image"
     key: "create-image"
     command: ".buildkite/scripts/build_push_docker_image.sh"
     agents:
       provider: "gcp"
+    plugins:
+      - *docker_elastic_login_plugin
 
   - label: "perf test"
     key: "obs-perf-test"

.buildkite/pipeline.yml

@@ -3,6 +3,25 @@
 env:
   DOCKER_COMPOSE_VERSION: "1.25.5"
   TERRAFORM_VERSION: "1.6.4"
+  IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-fleet-server-ubuntu-2204-fips-1751684469"
+
+# This section is used to define the plugins that will be used in the pipeline.
+# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
+common:
+  - test_collector_plugin: &test_collector_plugin
+      test-collector#v1.11.0:
+        files: "build/test-*.xml"
+        format: "junit"
+        branches: "main"
+        debug: true
+  - bk_analytics_token_plugin: &bk_analytics_token_plugin
+      elastic/vault-secrets#v0.1.0:
+        path: "kv/ci-shared/platform-ingest/buildkite_analytics_token"
+        field: "token"
+        env_var: "BUILDKITE_ANALYTICS_TOKEN"
+  - docker_elastic_login_plugin: &docker_elastic_login_plugin
+      elastic/vault-docker-login#v0.6.0:
+        secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry'
 
 steps:
   - group: "Check and build"
@@ -81,24 +100,31 @@ steps:
           - build/*.xml
           - build/coverage*.out
 
-      - label: ":smartbear-testexecute: Run unit tests with requirefips build tag"
+      - label: ":smartbear-testexecute: Run unit tests with requirefips build tag and FIPS provider"
         key: unit-test-fips-tag
         command: ".buildkite/scripts/unit_test.sh"
         env:
           FIPS: "true"
+          GOEXPERIMENT: "systemcrypto"
+          GO_DISTRO: "microsoft"
         agents:
-          provider: "gcp"
+          provider: "aws"
+          image: "${IMAGE_UBUNTU_X86_64_FIPS}"
+          instanceType: "m5.xlarge"
         artifact_paths:
           - build/*.xml
           - build/coverage*.out
 
-      - label: ":smartbear-testexecute: Run fips140=only unit tests"
+      - label: ":smartbear-testexecute: Run fips140=only unit tests with FIPS provider"
         key: unit-test-fips140-only
         command: ".buildkite/scripts/unit_test_fipsonly.sh"
         env:
           FIPS: "true"
+          GO_DISTRO: "stdlib"
         agents:
-          provider: "gcp"
+          provider: "aws"
+          image: "${IMAGE_UBUNTU_X86_64_FIPS}"
+          instanceType: "m5.xlarge"
         artifact_paths:
           - build/*.xml
           - build/coverage*.out
@@ -166,7 +192,6 @@ steps:
       - label: ":gcloud: Cloud e2e Test"
         key: "cloud-e2e-test"
         env:
-          DOCKER_REGISTRY: "docker.elastic.co"
           DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fleet"
           DOCKER_IMAGE_TAG: "pr-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_COMMIT:0:12}"
           SNAPSHOT: "true"
@@ -175,6 +200,8 @@ steps:
         command: ".buildkite/scripts/cloud_e2e_test.sh"
         agents:
           provider: "gcp"
+        plugins:
+          - *docker_elastic_login_plugin
         depends_on:
           - step: "unit-test"
             allow_failure: false
@@ -191,7 +218,6 @@ steps:
       - label: ":gcloud: Cloud e2e FIPS Test"
         key: "cloud-e2e-fips-test"
         env:
-          DOCKER_REGISTRY: "docker.elastic.co"
           DOCKER_BASE_IMAGE: "docker.elastic.co/cloud-release/elastic-agent-cloud-fips"
           DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips"
           DOCKER_IMAGE_TAG: "pr-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_COMMIT:0:12}"
@@ -202,6 +228,8 @@ steps:
         command: ".buildkite/scripts/cloud_e2e_test.sh"
         agents:
           provider: "gcp"
+        plugins:
+          - *docker_elastic_login_plugin
         depends_on:
           - step: "unit-test"
             allow_failure: false
@@ -219,14 +247,15 @@ steps:
     key: "publish"
     command: ".buildkite/scripts/build_push_docker_image.sh"
     env:
-      DOCKER_REGISTRY: "docker.elastic.co"
       DOCKER_IMAGE: "docker.elastic.co/observability-ci/fleet-server" # needs to rename for rollback
       DOCKER_IMAGE_SHA_TAG: "git-${BUILDKITE_COMMIT:0:12}" # needs to rename for rollback, should be "git-${BUILDKITE_COMMIT:0:12}"
       DOCKER_IMAGE_LATEST_TAG: "latest" # needs to rename for rollback
       DOCKER_IMAGE_GIT_TAG: "${BUILDKITE_BRANCH}" # needs to rename for rollback
     if: "build.env('BUILDKITE_PULL_REQUEST') == 'false' && build.env('BUILDKITE_BRANCH') == 'main'"
     agents:
       provider: "gcp"
+    plugins:
+      - *docker_elastic_login_plugin
     depends_on:
       - step: "tests"
         allow_failure: false

.buildkite/scripts/common.sh

@@ -55,9 +55,12 @@ with_msft_go() {
     echo "Setting up microsoft/go"
     create_workspace
     check_platform_architeture
+
+    # Use a temporary folder to house the Go SDK downloaded from Microsoft
+    tempfolder=$(mktemp -d)
     MSFT_DOWNLOAD_URL=https://aka.ms/golang/release/latest/go$(cat .go-version).${platform_type}-${arch_type}.tar.gz
-    retry 5 $(curl -sL -o - $MSFT_DOWNLOAD_URL | tar -xz -f - -C ${WORKSPACE})
-    export PATH="${PATH}:${WORKSPACE}/go/bin"
+    retry 5 $(curl -sL -o - $MSFT_DOWNLOAD_URL | tar -xz -f - -C ${tempfolder}/)
+    export PATH="${PATH}:${tempfolder}/go/bin"
     go version
     which go
     export PATH="${PATH}:$(go env GOPATH)/bin"
@@ -90,11 +93,6 @@ retry() {
     return 0
 }
 
-docker_logout() {
-    echo "Logging out from Docker..."
-    docker logout ${DOCKER_REGISTRY}
-}
-
 with_Terraform() {
     echo "Setting up the Terraform environment..."
     local path_to_file="${WORKSPACE}/terraform.zip"

.buildkite/scripts/unit_test.sh

@@ -6,7 +6,11 @@ source .buildkite/scripts/common.sh
 
 add_bin_path
 
-with_go
+if [[ ${FIPS:-false} == "true" && ${GO_DISTRO:-stdlib} == "microsoft" ]]; then
+    with_msft_go
+else
+  with_go
+fi
 
 with_mage
 

.buildkite/scripts/unit_test_fipsonly.sh

@@ -6,7 +6,11 @@ source .buildkite/scripts/common.sh
 
 add_bin_path
 
-with_go
+if [[ ${FIPS:-false} == "true" && ${GO_DISTRO:-stdlib} == "microsoft" ]]; then
+    with_msft_go
+else
+  with_go
+fi
 
 with_mage