[8.18] backport #4912 #4985 #5000 #5075 (#5397)

* Convert Makefile to magefile.go (#4912)

Reimplement Makefile targets in mage.
Change buildkite pipeline to use mage.
Change Dockerfile* to use mage.

(cherry picked from commit db5f46b)

# Conflicts:
#	.buildkite/pipeline.yml
#	.buildkite/scripts/common.sh
#	.buildkite/scripts/local_build.sh
#	.buildkite/scripts/test-release.sh
#	Dockerfile.build
#	Dockerfile.fips
#	Makefile
#	NOTICE.txt
#	README.md
#	dev-tools/cloud/Makefile
#	dev-tools/cloud/terraform/main.tf
#	dev-tools/e2e/Dockerfile
#	dev-tools/e2e/build.sh
#	dev-tools/e2e/validatecerts.go
#	dev-tools/go.mod
#	dev-tools/go.sum
#	docs/developers-guide.md
#	docs/docker-images.md
#	docs/fips.md
#	testing/e2e/fips_test.go

* Ensure directory is made within zip artifacts (#4985)

* Ensure directory is made within zip artifacts

* Make test:release extraction stricter

* Fail cloude2e step if tests can't run, fix image publication (#5000)

Fail the pipeline if cloude2e tests can't run.
mage docker:image now uses the DOCKER_IMAGE env var.

* Add multiplatform images (#5075)

Fixes issue where change from make to mage, result in the loss of creating multiple platform images.

* Fix cherry-picks

* change to serialdeps

* Use magefile from main

* Remove FIPS references, use docker plugin

* Use bk plugins

* cleanup go.mod in dev-tools and pkg/api

* Add missing EC secret to cloud e2e step

* Review feedback

Author: michel-laterman

.buildkite/hooks/pre-command

@@ -27,7 +27,7 @@ check_if_file_exist_in_repo() {
     local response=$(curl --fail -s -H "Authorization: token $GITHUB_REPO_TOKEN" https://api.github.com/repos/elastic/${repoName}/contents/${path_to_file}| grep -c "\"path\"\: \"${path_to_file}\"")
     if [[ ${response} -ge 1 ]]; then
         export FILE_EXISTS_IN_REPO=true
-        echo "FILE_EXIST_IN_REPO = true"        
+        echo "FILE_EXIST_IN_REPO = true"
     else
         export FILE_EXISTS_IN_REPO=false
         echo "FILE_EXIST_IN_REPO = false"
@@ -45,25 +45,8 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
   check_if_file_exist_in_repo "infra" "${_branch}"                  #TODO should be changed to "main" for rollback...
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" || "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-perf-tests" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "publish" || "$BUILDKITE_STEP_KEY" == "cloud-e2e-test" || "$BUILDKITE_STEP_KEY" == "create-image" ]]; then
-    export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
-    export DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
-    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
-  fi
-fi
-
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "cloud-e2e-test" ]]; then
-  export EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey "${EC_KEY_SECRET_PATH}")
-  # Environment variables required by the Elastic Cloud service deployer
-  export EC_API_KEY=${EC_API_KEY_SECRET}
-fi
-
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
   if [[ "$BUILDKITE_STEP_KEY" == "dra-snapshot" || "$BUILDKITE_STEP_KEY" == "dra-staging" ]]; then
-    export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
-    export DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
-    docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
     DRA_CREDS_SECRET=$(retry 5 vault kv get -field=data -format=json ${CI_DRA_ROLE_PATH})
     export VAULT_ADDR_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.vault_addr')
     export VAULT_ROLE_ID_SECRET=$(echo ${DRA_CREDS_SECRET} | jq -r '.role_id')

.buildkite/pipeline.yml

@@ -2,19 +2,27 @@
 
 env:
   DOCKER_COMPOSE_VERSION: "1.25.5"
-  DOCKER_REGISTRY: "docker.elastic.co"
-  DOCKER_IMAGE: "${DOCKER_REGISTRY}/observability-ci/fleet-server" # needs to rename for rollback
-  DOCKER_IMAGE_SHA_TAG: "git-${BUILDKITE_COMMIT:0:12}" # needs to rename for rollback, should be "git-${BUILDKITE_COMMIT:0:12}"
-  DOCKER_IMAGE_LATEST_TAG: "latest" # needs to rename for rollback
-  DOCKER_IMAGE_GIT_TAG: "${BUILDKITE_BRANCH}" # needs to rename for rollback
-  GO_AGENT_IMAGE: "golang:${GO_VERSION}"
   TERRAFORM_VERSION: "1.6.4"
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2204-aarch64"
 
 # This section is used to define the plugins that will be used in the pipeline.
 # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
 common:
+  - test_collector_plugin: &test_collector_plugin
+      test-collector#v1.11.0:
+        files: "build/test-*.xml"
+        format: "junit"
+        branches: "main"
+        debug: true
+  - bk_analytics_token_plugin: &bk_analytics_token_plugin
+      elastic/vault-secrets#v0.1.0:
+        path: "kv/ci-shared/platform-ingest/buildkite_analytics_token"
+        field: "token"
+        env_var: "BUILDKITE_ANALYTICS_TOKEN"
+  - docker_elastic_login_plugin: &docker_elastic_login_plugin
+      elastic/vault-docker-login#v0.6.0:
+        secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry'
   # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/fleet-server/01-gcp-buildkite-oidc.tf
   # This plugin authenticates to Google Cloud using the OIDC token.
   - oidc_plugin: &oidc_plugin
@@ -29,11 +37,9 @@ steps:
     steps:
       - label: ":white_check_mark: Run check-ci"
         key: check-ci
-        command: "make check-ci"
+        command: ".buildkite/scripts/check_ci.sh"
         agents:
-          image: "${GO_AGENT_IMAGE}"
-          cpu: "8"
-          memory: "4G"
+          provider: "gcp"
 
       - label: "Package x86_64"
         key: "package-x86-64-pr"
@@ -69,6 +75,8 @@ steps:
       - label: "Run go benchmark for PR branch"
         key: "go-benchmark-pr"
         command: ".buildkite/scripts/run_benchmark.sh pr"
+        env:
+          BENCHMARK_ARGS: "-count=8 -benchmem"
         artifact_paths:
           - build/next.out
           - build/next.stat
@@ -77,8 +85,11 @@ steps:
           machineType: "c2-standard-8"
 
       - label: "Run go benchmark for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
+        skip: true # FIXME: Temporarily disabled until mage is on main
         key: "go-benchmark-base"
         command: ".buildkite/scripts/run_benchmark.sh base"
+        env:
+          BENCHMARK_ARGS: "-count=8 -benchmem"
         artifact_paths:
           - build/base.out
           - build/base.stat
@@ -87,6 +98,7 @@ steps:
           machineType: "c2-standard-8"
 
       - label: "Compare results"
+        skip: true # FIXME: Temporarily disabled until mage is on main
         key: "go-benchmark-compare"
         command: ".buildkite/scripts/run_benchmark.sh compare"
         artifact_paths:
@@ -130,6 +142,9 @@ steps:
           provider: "gcp"
         artifact_paths:
           - build/*.xml
+        plugins:
+          - *bk_analytics_token_plugin
+          - *test_collector_plugin
         retry:
           automatic:
             limit: 1
@@ -144,6 +159,9 @@ steps:
         artifact_paths:
           - build/*.xml
           - build/e2e-coverage.out
+        plugins:
+          - *bk_analytics_token_plugin
+          - *test_collector_plugin
 
       - label: ":junit: Junit annotate"
         plugins:
@@ -180,9 +198,22 @@ steps:
 
       - label: ":gcloud: Cloud e2e Test"
         key: "cloud-e2e-test"
+        env:
+          DOCKER_REGISTRY: "docker.elastic.co"
+          DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fleet"
+          DOCKER_IMAGE_TAG: "pr-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_COMMIT:0:12}"
+          SNAPSHOT: "true"
+          PLATFORMS: "linux/amd64"
+          TF_VAR_pull_request: "${BUILDKITE_PULL_REQUEST}"
         command: ".buildkite/scripts/cloud_e2e_test.sh"
         agents:
           provider: "gcp"
+        plugins:
+          - *docker_elastic_login_plugin
+          - elastic/vault-secrets#v0.1.0:
+              path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
+              field: "apiKey"
+              env_var: "EC_API_KEY"
         depends_on:
           - step: "unit-test"
             allow_failure: false
@@ -195,9 +226,16 @@ steps:
   - label: ":docker: Publish docker image"
     key: "publish"
     command: ".buildkite/scripts/build_push_docker_image.sh"
+    env:
+      DOCKER_IMAGE: "docker.elastic.co/observability-ci/fleet-server" # needs to rename for rollback
+      DOCKER_IMAGE_SHA_TAG: "git-${BUILDKITE_COMMIT:0:12}" # needs to rename for rollback, should be "git-${BUILDKITE_COMMIT:0:12}"
+      DOCKER_IMAGE_LATEST_TAG: "latest" # needs to rename for rollback
+      DOCKER_IMAGE_GIT_TAG: "${BUILDKITE_BRANCH}" # needs to rename for rollback
     if: "build.env('BUILDKITE_PULL_REQUEST') == 'false' && build.env('BUILDKITE_BRANCH') == 'main'"
     agents:
       provider: "gcp"
+    plugins:
+      - *docker_elastic_login_plugin
     depends_on:
       - step: "tests"
         allow_failure: false

.buildkite/scripts/build_push_docker_image.sh

@@ -4,14 +4,20 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
+add_bin_path
+
+with_go
+
+with_mage
+
 echo "Building the docker image..."
 if ! docker pull -q ${DOCKER_IMAGE}:${DOCKER_IMAGE_SHA_TAG} 2> /dev/null; then
     DOCKER_IMAGE_TAG="${DOCKER_IMAGE_SHA_TAG}"
-    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG} make build-and-push-docker
+    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG} mage docker:publish
 fi
 
 if [[ "${DOCKER_IMAGE_GIT_TAG}" == "main" ]]; then
-    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG="${DOCKER_IMAGE_LATEST_TAG}" make build-and-push-docker
+    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG="${DOCKER_IMAGE_LATEST_TAG}" mage docker:publish
 elif [[ ${BUILDKITE_PULL_REQUEST} == "false" ]]; then
-    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG="${DOCKER_IMAGE_GIT_TAG}" make build-and-push-docker
+    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG="${DOCKER_IMAGE_GIT_TAG}" mage docker:publish
 fi

.buildkite/scripts/check_ci.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common.sh
+
+add_bin_path
+
+with_go
+
+with_mage
+
+mage check:ci

.buildkite/scripts/cloud_e2e_test.sh

@@ -12,4 +12,37 @@ with_Terraform
 
 with_docker_compose
 
-USER=fleetserverci make test-cloude2e
+with_mage
+
+cleanup() {
+  r=$?
+
+  if [ -f dev-tools/cloud/terraform/.terraform.lock.hcl ] ; then
+    echo "--- Deployment detected, running cleanup."
+    mage test:cloudE2EDown
+  else
+      echo "--- No deployment detected, skipping cleanup."
+  fi
+  exit $r
+}
+trap cleanup EXIT INT TERM
+
+USER=fleetserverci mage docker:cover docker:customAgentImage docker:push test:cloudE2EUp
+FLEET_SERVER_URL=$(terraform output --raw --state=dev-tools/cloud/terraform/terraform.tfstate fleet_url)
+echo "Fleet server: \"${FLEET_SERVER_URL}\""
+echo "Deployment ID: $(terraform output --raw --state=dev-tools/cloud/terraform/terraform.tfstate deployment_id)"
+
+if [[ "${FLEET_SERVER_URL}" == "" ]]; then
+    message="FLEET_SERVER_URL is empty, cloud e2e tests cannot be executed"
+    if [[ "${CI}" == "true" ]]; then
+        buildkite-agent annotate \
+            "${message}" \
+            --context "ctx-cloude2e-test" \
+            --style "error"
+    fi
+    echo "${message}"
+    exit 1
+fi
+
+echo "--- Trigger cloud E2E test"
+mage test:cloudE2ERun

.buildkite/scripts/common.sh

@@ -136,21 +136,15 @@ download_mbp_packages_from_gcp_bucket() {
 }
 
 with_mage() {
-    local install_packages=(
-            "github.com/magefile/mage"
-            "github.com/elastic/go-licenser"
-            "golang.org/x/tools/cmd/goimports"
-            "github.com/jstemmer/go-junit-report"
-            "gotest.tools/gotestsum"
-    )
     create_workspace
-    for pkg in "${install_packages[@]}"; do
-        go install "${pkg}@latest"
-    done
+    go install github.com/magefile/mage # uses go.mod implicitly
+    mage -clean
+    mage -version
+    which mage
 }
 
 cleanup() {
     echo "Deleting temporary files..."
     rm -rf ${WORKSPACE}/${TMP_FOLDER_TEMPLATE_BASE}.*
     echo "Done."
-}
+}

.buildkite/scripts/dra_release.sh

@@ -4,6 +4,12 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
+add_bin_path
+
+with_go
+
+with_mage
+
 FOLDER_PATH="build/distributions"
 BASE_DIR="${WORKSPACE}/${FOLDER_PATH}"
 DRA_OUTPUT="release-manager.out"
@@ -12,7 +18,7 @@ export TYPE=${1}
 # DRA_BRANCH can be used for manually testing packaging with PRs
 # e.g. define `DRA_BRANCH="main"` under Options/Environment Variables in the Buildkite UI after clicking new Build
 export BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
-export VERSION="$(make get-version)"
+export VERSION="$(mage getVersion)"
 
 if [[ "${VERSION}" == *"-SNAPSHOT"* || "${VERSION}" == "" ]]; then
     echo "The 'version' parameter is required and it cannot contain the suffix '-SNAPSHOT'."
@@ -24,14 +30,7 @@ if [[ "${PROJECT}" == "" ]]; then
     exit 1
 fi
 
-add_bin_path
-
 download_mbp_packages_from_gcp_bucket "${FOLDER_PATH}" "${TYPE}"
-
-with_go
-
-with_mage
-
 export RM_VERSION="${VERSION}"
 
 if [[ ${TYPE} == "snapshot" ]]; then

.buildkite/scripts/e2e_test.sh

@@ -10,5 +10,7 @@ with_go
 
 with_docker_compose
 
+with_mage
+
 echo "Starting the E2E tests..."
-make test-e2e junit-report
+mage test:e2e test:junitReport

.buildkite/scripts/integration_test.sh

@@ -10,5 +10,7 @@ with_go
 
 with_docker_compose
 
+with_mage
+
 echo "Starting the integration tests..."
-make test-int junit-report
+mage test:integration test:junitReport

.buildkite/scripts/local_build.sh

@@ -7,4 +7,6 @@ source .buildkite/scripts/common.sh
 add_bin_path
 with_go
 
-make local
+with_mage
+
+mage build:local