[9.0] backport #4912 #4985 #5000 #5075

.buildkite/pipeline.yml

@@ -2,12 +2,6 @@
 
 env:
   DOCKER_COMPOSE_VERSION: "1.25.5"
-  DOCKER_REGISTRY: "docker.elastic.co"
-  DOCKER_IMAGE: "${DOCKER_REGISTRY}/observability-ci/fleet-server" # needs to rename for rollback
-  DOCKER_IMAGE_SHA_TAG: "git-${BUILDKITE_COMMIT:0:12}" # needs to rename for rollback, should be "git-${BUILDKITE_COMMIT:0:12}"
-  DOCKER_IMAGE_LATEST_TAG: "latest" # needs to rename for rollback
-  DOCKER_IMAGE_GIT_TAG: "${BUILDKITE_BRANCH}" # needs to rename for rollback
-  GO_AGENT_IMAGE: "golang:${GO_VERSION}"
   TERRAFORM_VERSION: "1.6.4"
   IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
   IMAGE_UBUNTU_ARM_64: "core-ubuntu-2204-aarch64"
@@ -29,11 +23,9 @@ steps:
     steps:
       - label: ":white_check_mark: Run check-ci"
         key: check-ci
-        command: "make check-ci"
+        command: ".buildkite/scripts/check_ci.sh"
         agents:
-          image: "${GO_AGENT_IMAGE}"
-          cpu: "8"
-          memory: "4G"
+          provider: "gcp"
 
       - label: "Package x86_64"
         key: "package-x86-64-pr"
@@ -69,6 +61,8 @@ steps:
       - label: "Run go benchmark for PR branch"
         key: "go-benchmark-pr"
         command: ".buildkite/scripts/run_benchmark.sh pr"
+        env:
+          BENCHMARK_ARGS: "-count=8 -benchmem"
         artifact_paths:
           - build/next.out
           - build/next.stat
@@ -77,8 +71,11 @@ steps:
           machineType: "c2-standard-8"
 
       - label: "Run go benchmark for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
+        skip: true # FIXME: Temporarily disabled until mage is on main
         key: "go-benchmark-base"
         command: ".buildkite/scripts/run_benchmark.sh base"
+        env:
+          BENCHMARK_ARGS: "-count=8 -benchmem"
         artifact_paths:
           - build/base.out
           - build/base.stat
@@ -87,6 +84,7 @@ steps:
           machineType: "c2-standard-8"
 
       - label: "Compare results"
+        skip: true # FIXME: Temporarily disabled until mage is on main
         key: "go-benchmark-compare"
         command: ".buildkite/scripts/run_benchmark.sh compare"
         artifact_paths:
@@ -203,6 +201,13 @@ steps:
 
       - label: ":gcloud: Cloud e2e Test"
         key: "cloud-e2e-test"
+        env:
+          DOCKER_REGISTRY: "docker.elastic.co"
+          DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fleet"
+          DOCKER_IMAGE_TAG: "pr-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_COMMIT:0:12}"
+          SNAPSHOT: "true"
+          PLATFORMS: "linux/amd64"
+          TF_VAR_pull_request: "${BUILDKITE_PULL_REQUEST}"
         command: ".buildkite/scripts/cloud_e2e_test.sh"
         agents:
           provider: "gcp"
@@ -218,6 +223,12 @@ steps:
   - label: ":docker: Publish docker image"
     key: "publish"
     command: ".buildkite/scripts/build_push_docker_image.sh"
+    env:
+      DOCKER_REGISTRY: "docker.elastic.co"
+      DOCKER_IMAGE: "docker.elastic.co/observability-ci/fleet-server" # needs to rename for rollback
+      DOCKER_IMAGE_SHA_TAG: "git-${BUILDKITE_COMMIT:0:12}" # needs to rename for rollback, should be "git-${BUILDKITE_COMMIT:0:12}"
+      DOCKER_IMAGE_LATEST_TAG: "latest" # needs to rename for rollback
+      DOCKER_IMAGE_GIT_TAG: "${BUILDKITE_BRANCH}" # needs to rename for rollback
     if: "build.env('BUILDKITE_PULL_REQUEST') == 'false' && build.env('BUILDKITE_BRANCH') == 'main'"
     agents:
       provider: "gcp"

.buildkite/scripts/build_push_docker_image.sh

@@ -4,14 +4,20 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
+add_bin_path
+
+with_go
+
+with_mage
+
 echo "Building the docker image..."
 if ! docker pull -q ${DOCKER_IMAGE}:${DOCKER_IMAGE_SHA_TAG} 2> /dev/null; then
     DOCKER_IMAGE_TAG="${DOCKER_IMAGE_SHA_TAG}"
-    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG} make build-and-push-docker
+    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG} mage docker:publish
 fi
 
 if [[ "${DOCKER_IMAGE_GIT_TAG}" == "main" ]]; then
-    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG="${DOCKER_IMAGE_LATEST_TAG}" make build-and-push-docker
+    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG="${DOCKER_IMAGE_LATEST_TAG}" mage docker:publish
 elif [[ ${BUILDKITE_PULL_REQUEST} == "false" ]]; then
-    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG="${DOCKER_IMAGE_GIT_TAG}" make build-and-push-docker
+    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG="${DOCKER_IMAGE_GIT_TAG}" mage docker:publish
 fi

.buildkite/scripts/check_ci.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common.sh
+
+add_bin_path
+
+with_go
+
+with_mage
+
+mage check:ci

.buildkite/scripts/cloud_e2e_test.sh

@@ -12,4 +12,37 @@ with_Terraform
 
 with_docker_compose
 
-USER=fleetserverci make test-cloude2e
+with_mage
+
+cleanup() {
+  r=$?
+
+  if [ -f dev-tools/cloud/terraform/.terraform.lock.hcl ] ; then
+    echo "--- Deployment detected, running cleanup."
+    mage test:cloudE2EDown
+  else
+      echo "--- No deployment detected, skipping cleanup."
+  fi
+  exit $r
+}
+trap cleanup EXIT INT TERM
+
+USER=fleetserverci mage docker:cover docker:customAgentImage docker:push test:cloudE2EUp
+FLEET_SERVER_URL=$(terraform output --raw --state=dev-tools/cloud/terraform/terraform.tfstate fleet_url)
+echo "Fleet server: \"${FLEET_SERVER_URL}\""
+echo "Deployment ID: $(terraform output --raw --state=dev-tools/cloud/terraform/terraform.tfstate deployment_id)"
+
+if [[ "${FLEET_SERVER_URL}" == "" ]]; then
+    message="FLEET_SERVER_URL is empty, cloud e2e tests cannot be executed"
+    if [[ "${CI}" == "true" ]]; then
+        buildkite-agent annotate \
+            "${message}" \
+            --context "ctx-cloude2e-test" \
+            --style "error"
+    fi
+    echo "${message}"
+    exit 1
+fi
+
+echo "--- Trigger cloud E2E test"
+mage test:cloudE2ERun

.buildkite/scripts/common.sh

@@ -136,21 +136,15 @@ download_mbp_packages_from_gcp_bucket() {
 }
 
 with_mage() {
-    local install_packages=(
-            "github.com/magefile/mage"
-            "github.com/elastic/go-licenser"
-            "golang.org/x/tools/cmd/goimports"
-            "github.com/jstemmer/go-junit-report"
-            "gotest.tools/gotestsum"
-    )
     create_workspace
-    for pkg in "${install_packages[@]}"; do
-        go install "${pkg}@latest"
-    done
+    go install github.com/magefile/mage # uses go.mod implicitly
+    mage -clean
+    mage -version
+    which mage
 }
 
 cleanup() {
     echo "Deleting temporary files..."
     rm -rf ${WORKSPACE}/${TMP_FOLDER_TEMPLATE_BASE}.*
     echo "Done."
-}
+}

.buildkite/scripts/dra_release.sh

@@ -4,6 +4,12 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
+add_bin_path
+
+with_go
+
+with_mage
+
 FOLDER_PATH="build/distributions"
 BASE_DIR="${WORKSPACE}/${FOLDER_PATH}"
 DRA_OUTPUT="release-manager.out"
@@ -12,7 +18,7 @@ export TYPE=${1}
 # DRA_BRANCH can be used for manually testing packaging with PRs
 # e.g. define `DRA_BRANCH="main"` under Options/Environment Variables in the Buildkite UI after clicking new Build
 export BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
-export VERSION="$(make get-version)"
+export VERSION="$(mage getVersion)"
 
 if [[ "${VERSION}" == *"-SNAPSHOT"* || "${VERSION}" == "" ]]; then
     echo "The 'version' parameter is required and it cannot contain the suffix '-SNAPSHOT'."
@@ -24,14 +30,7 @@ if [[ "${PROJECT}" == "" ]]; then
     exit 1
 fi
 
-add_bin_path
-
 download_mbp_packages_from_gcp_bucket "${FOLDER_PATH}" "${TYPE}"
-
-with_go
-
-with_mage
-
 export RM_VERSION="${VERSION}"
 
 if [[ ${TYPE} == "snapshot" ]]; then

.buildkite/scripts/e2e_test.sh

@@ -10,5 +10,7 @@ with_go
 
 with_docker_compose
 
+with_mage
+
 echo "Starting the E2E tests..."
-make test-e2e junit-report
+mage test:e2e test:junitReport

.buildkite/scripts/integration_test.sh

@@ -10,5 +10,7 @@ with_go
 
 with_docker_compose
 
+with_mage
+
 echo "Starting the integration tests..."
-make test-int junit-report
+mage test:integration test:junitReport

.buildkite/scripts/local_build.sh

@@ -7,4 +7,6 @@ source .buildkite/scripts/common.sh
 add_bin_path
 with_go
 
-make local
+with_mage
+
+mage build:local

.buildkite/scripts/package.sh

@@ -27,10 +27,10 @@ with_mage
 case "${TYPE}" in
     "snapshot")
         export SNAPSHOT=true
-        make docker-release
+        mage docker:release
         ;;
     "staging")
-        make docker-release
+        mage docker:release
         ;;
     *)
     echo "The option is unsupported yet"

.buildkite/scripts/release_test.sh

@@ -14,10 +14,12 @@ add_bin_path
 
 with_go
 
-make docker-release
+with_mage
+
+mage docker:release
 
 upload_packages_to_gcp_bucket "build/distributions/"
 
-make test-release
+mage test:release
 
-make build-docker
+mage docker:image

.buildkite/scripts/run_benchmark.sh

@@ -8,14 +8,14 @@ add_bin_path
 
 with_go
 
+with_mage
+
 export TYPE=${1}
-#export BRANCH="${BUILDKITE_BRANCH}"
-export BENCHMARK_ARGS="-count=8 -benchmem"
 
 if [[ ${TYPE} == "pr" ]]; then
     echo "Starting the go benchmark for the pull request"
-    BENCH_BASE=next.out make benchmark
-    BENCH_BASE=next.out make benchstat | tee build/next.stat
+    BENCH_BASE=next.out mage test:benchmark
+    BENCH_BASE=next.out mage test:benchstat | tee build/next.stat
     BENCH=$(cat build/next.out)
     buildkite-agent annotate --style 'info' --context "gobench_pr" --append << _EOF_
 #### Benchmark for pull request
@@ -34,8 +34,8 @@ fi
 if [[ ${TYPE} == "base" ]]; then
     echo "Starting the go benchmark for the pull request"
     git checkout ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
-    BENCH_BASE=base.out make benchmark
-    BENCH_BASE=base.out make benchstat | tee build/base.stat
+    BENCH_BASE=base.out mage test:benchmark
+    BENCH_BASE=base.out mage test:benchstat | tee build/base.stat
     BENCH=$(cat build/base.out)
     buildkite-agent annotate --style 'info' --context "gobench_base" --append << _EOF_
 #### Benchmark for the ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
@@ -93,5 +93,3 @@ Download <a href="artifact://build/full_report.json">full_report.json</a>
 _EOF_
     fi
 fi
-
-

.buildkite/scripts/unit_test.sh

@@ -8,9 +8,7 @@ add_bin_path
 
 with_go
 
+with_mage
+
 echo "Starting the unit tests..."
-if [[ ${FIPS:-} == "true" ]]; then
-    make test-unit-fips junit-report
-else
-    make test-unit junit-report
-fi
+mage test:unit test:junitReport