[8.19](backport #4912) Convert Makefile to magefile.go (#4981)

* Convert Makefile to magefile.go (#4912)

Reimplement Makefile targets in mage.
Change buildkite pipeline to use mage.
Change Dockerfile* to use mage.

(cherry picked from commit db5f46b)

# Conflicts:
#	Dockerfile.build
#	Dockerfile.fips
#	Makefile
#	NOTICE.txt
#	README.md
#	dev-tools/go.mod
#	dev-tools/go.sum
#	docs/developers-guide.md
#	docs/docker-images.md

* Clean up backport

* Delete .terraform.lock.hcl

* Fix check:ci

* Fix imports

---------

Co-authored-by: Michel Laterman <82832767+michel-laterman@users.noreply.github.com>
Co-authored-by: michel-laterman <michel.laterman@elastic.co>

Author: mergify[bot]

.buildkite/pipeline.yml

@@ -2,12 +2,6 @@
 
 env:
   DOCKER_COMPOSE_VERSION: "1.25.5"
-  DOCKER_REGISTRY: "docker.elastic.co"
-  DOCKER_IMAGE: "${DOCKER_REGISTRY}/observability-ci/fleet-server" # needs to rename for rollback
-  DOCKER_IMAGE_SHA_TAG: "git-${BUILDKITE_COMMIT:0:12}" # needs to rename for rollback, should be "git-${BUILDKITE_COMMIT:0:12}"
-  DOCKER_IMAGE_LATEST_TAG: "latest" # needs to rename for rollback
-  DOCKER_IMAGE_GIT_TAG: "${BUILDKITE_BRANCH}" # needs to rename for rollback
-  GO_AGENT_IMAGE: "golang:${GO_VERSION}"
   TERRAFORM_VERSION: "1.6.4"
 
 steps:
@@ -16,19 +10,15 @@ steps:
     steps:
       - label: ":white_check_mark: Run check-ci"
         key: check-ci
-        command: "make check-ci"
+        command: ".buildkite/scripts/check_ci.sh"
         agents:
-          image: "${GO_AGENT_IMAGE}"
-          cpu: "8"
-          memory: "4G"
+          provider: "gcp"
 
       - label: ":building_construction: Run local build"
         key: make-local
-        command: "make local"
+        command: ".buildkite/scripts/local_build.sh"
         agents:
-          image: "${GO_AGENT_IMAGE}"
-          cpu: "8"
-          memory: "4G"
+          provider: "gcp"
 
       - label: ":building_construction: Run local build: MacOS"
         key: make-local-macos
@@ -43,6 +33,8 @@ steps:
       - label: "Run go benchmark for PR branch"
         key: "go-benchmark-pr"
         command: ".buildkite/scripts/run_benchmark.sh pr"
+        env:
+          BENCHMARK_ARGS: "-count=8 -benchmem"
         artifact_paths:
           - build/next.out
           - build/next.stat
@@ -51,8 +43,11 @@ steps:
           machineType: "c2-standard-8"
 
       - label: "Run go benchmark for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
+        skip: true # FIXME: Temporarily disabled until mage is on main
         key: "go-benchmark-base"
         command: ".buildkite/scripts/run_benchmark.sh base"
+        env:
+          BENCHMARK_ARGS: "-count=8 -benchmem"
         artifact_paths:
           - build/base.out
           - build/base.stat
@@ -61,6 +56,7 @@ steps:
           machineType: "c2-standard-8"
 
       - label: "Compare results"
+        skip: true # FIXME: Temporarily disabled until mage is on main
         key: "go-benchmark-compare"
         command: ".buildkite/scripts/run_benchmark.sh compare"
         artifact_paths:
@@ -100,10 +96,9 @@ steps:
 
       - label: ":smartbear-testexecute: Run fips140=only unit tests"
         key: unit-test-fips140-only
-        command: ".buildkite/scripts/unit_test.sh"
+        command: ".buildkite/scripts/unit_test_fipsonly.sh"
         env:
           FIPS: "true"
-          GODEBUG: "fips140=only"
         agents:
           provider: "gcp"
         artifact_paths:
@@ -137,23 +132,6 @@ steps:
           - build/*.xml
           - build/e2e-coverage.out
 
-      - label: "FIPS E2E Test"
-        key: "fips-e2e-test"
-        env:
-          FIPS: true
-        command: ".buildkite/scripts/e2e_test.sh"
-        agents:
-          provider: "gcp"
-        artifact_paths:
-          - build/*.xml
-          - build/e2e-coverage.out
-        plugins:
-          - test-collector#v1.10.2:
-              files: "build/test-*.xml"
-              format: "junit"
-              branches: "main"
-              debug: true
-
       - label: ":junit: Junit annotate"
         plugins:
           - junit-annotate#v2.4.1:
@@ -189,6 +167,13 @@ steps:
 
       - label: ":gcloud: Cloud e2e Test"
         key: "cloud-e2e-test"
+        env:
+          DOCKER_REGISTRY: "docker.elastic.co"
+          DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fleet"
+          DOCKER_IMAGE_TAG: "pr-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_COMMIT:0:12}"
+          SNAPSHOT: "true"
+          PLATFORMS: "linux/amd64"
+          TF_VAR_pull_request: "${BUILDKITE_PULL_REQUEST}"
         command: ".buildkite/scripts/cloud_e2e_test.sh"
         agents:
           provider: "gcp"
@@ -204,6 +189,12 @@ steps:
   - label: ":docker: Publish docker image"
     key: "publish"
     command: ".buildkite/scripts/build_push_docker_image.sh"
+    env:
+      DOCKER_REGISTRY: "docker.elastic.co"
+      DOCKER_IMAGE: "docker.elastic.co/observability-ci/fleet-server" # needs to rename for rollback
+      DOCKER_IMAGE_SHA_TAG: "git-${BUILDKITE_COMMIT:0:12}" # needs to rename for rollback, should be "git-${BUILDKITE_COMMIT:0:12}"
+      DOCKER_IMAGE_LATEST_TAG: "latest" # needs to rename for rollback
+      DOCKER_IMAGE_GIT_TAG: "${BUILDKITE_BRANCH}" # needs to rename for rollback
     if: "build.env('BUILDKITE_PULL_REQUEST') == 'false' && build.env('BUILDKITE_BRANCH') == 'main'"
     agents:
       provider: "gcp"

.buildkite/scripts/build_push_docker_image.sh

@@ -4,14 +4,20 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
+add_bin_path
+
+with_go
+
+with_mage
+
 echo "Building the docker image..."
 if ! docker pull -q ${DOCKER_IMAGE}:${DOCKER_IMAGE_SHA_TAG} 2> /dev/null; then
     DOCKER_IMAGE_TAG="${DOCKER_IMAGE_SHA_TAG}"
-    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG} make build-and-push-docker
+    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG} mage docker:image docker:push
 fi
 
 if [[ "${DOCKER_IMAGE_GIT_TAG}" == "main" ]]; then
-    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG="${DOCKER_IMAGE_LATEST_TAG}" make build-and-push-docker
+    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG="${DOCKER_IMAGE_LATEST_TAG}" mage docker:image docker:push
 elif [[ ${BUILDKITE_PULL_REQUEST} == "false" ]]; then
-    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG="${DOCKER_IMAGE_GIT_TAG}" make build-and-push-docker
+    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG="${DOCKER_IMAGE_GIT_TAG}" mage docker:image docker:push
 fi

.buildkite/scripts/check_ci.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common.sh
+
+add_bin_path
+
+with_go
+
+with_mage
+
+mage check:ci

.buildkite/scripts/cloud_e2e_test.sh

@@ -12,4 +12,37 @@ with_Terraform
 
 with_docker_compose
 
-USER=fleetserverci make test-cloude2e
+with_mage
+
+cleanup() {
+  r=$?
+
+  if [ -f dev-tools/cloud/terraform/.terraform.lock.hcl ] ; then
+    echo "--- Deployment detected, running cleanup."
+    mage test:cloudE2EDown
+  else
+      echo "--- No deployment detected, skipping cleanup."
+  fi
+  exit $r
+}
+trap cleanup EXIT INT TERM
+
+USER=fleetserverci mage docker:cover docker:customAgentImage docker:push test:cloudE2EUp
+FLEET_SERVER_URL=$(terraform output --raw --state=dev-tools/cloud/terraform/terraform.tfstate fleet_url)
+echo "Fleet server: \"${FLEET_SERVER_URL}\""
+echo "Deployment ID: $(terraform output --raw --state=dev-tools/cloud/terraform/terraform.tfstate deployment_id)"
+
+if [[ "${FLEET_SERVER_URL}" == "" ]]; then
+    message="FLEET_SERVER_URL is empty, cloud e2e tests cannot be executed"
+    if [[ "${CI}" == "true" ]]; then
+        buildkite-agent annotate \
+            "${message}" \
+            --context "ctx-cloude2e-test" \
+            --style "error"
+    fi
+    echo "${message}"
+    exit 0
+fi
+
+echo "--- Trigger cloud E2E test"
+mage test:cloudE2ERun

.buildkite/scripts/common.sh

@@ -55,7 +55,7 @@ with_msft_go() {
     echo "Setting up microsoft/go"
     create_workspace
     check_platform_architeture
-    MSFT_DOWNLOAD_URL=https://aka.ms/golang/release/latest/go$(cat .go-version)-1.${platform_type}-${arch_type}.tar.gz
+    MSFT_DOWNLOAD_URL=https://aka.ms/golang/release/latest/go$(cat .go-version).${platform_type}-${arch_type}.tar.gz
     retry 5 $(curl -sL -o - $MSFT_DOWNLOAD_URL | tar -xz -f - -C ${WORKSPACE})
     export PATH="${PATH}:${WORKSPACE}/go/bin"
     go version
@@ -148,17 +148,11 @@ download_mbp_packages_from_gcp_bucket() {
 }
 
 with_mage() {
-    local install_packages=(
-            "github.com/magefile/mage"
-            "github.com/elastic/go-licenser"
-            "golang.org/x/tools/cmd/goimports"
-            "github.com/jstemmer/go-junit-report"
-            "gotest.tools/gotestsum"
-    )
     create_workspace
-    for pkg in "${install_packages[@]}"; do
-        go install "${pkg}@latest"
-    done
+    go install github.com/magefile/mage # uses go.mod implicitly
+    mage -clean
+    mage -version
+    which mage
 }
 
 cleanup() {

.buildkite/scripts/dra_release.sh

@@ -4,6 +4,12 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
+add_bin_path
+
+with_go
+
+with_mage
+
 FOLDER_PATH="build/distributions"
 BASE_DIR="${WORKSPACE}/${FOLDER_PATH}"
 DRA_OUTPUT="release-manager.out"
@@ -12,7 +18,7 @@ export TYPE=${1}
 # DRA_BRANCH can be used for manually testing packaging with PRs
 # e.g. define `DRA_BRANCH="main"` under Options/Environment Variables in the Buildkite UI after clicking new Build
 export BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
-export VERSION="$(make get-version)"
+export VERSION="$(mage getVersion)"
 
 if [[ "${VERSION}" == *"-SNAPSHOT"* || "${VERSION}" == "" ]]; then
     echo "The 'version' parameter is required and it cannot contain the suffix '-SNAPSHOT'."
@@ -24,14 +30,7 @@ if [[ "${PROJECT}" == "" ]]; then
     exit 1
 fi
 
-add_bin_path
-
 download_mbp_packages_from_gcp_bucket "${FOLDER_PATH}" "${TYPE}"
-
-with_go
-
-with_mage
-
 export RM_VERSION="${VERSION}"
 
 if [[ ${TYPE} == "snapshot" ]]; then

.buildkite/scripts/e2e_test.sh

@@ -10,5 +10,7 @@ with_go
 
 with_docker_compose
 
+with_mage
+
 echo "Starting the E2E tests..."
-make test-e2e junit-report
+mage test:e2e test:junitReport

.buildkite/scripts/integration_test.sh

@@ -10,5 +10,7 @@ with_go
 
 with_docker_compose
 
+with_mage
+
 echo "Starting the integration tests..."
-make test-int junit-report
+mage test:integration test:junitReport

.buildkite/scripts/local_build.sh

@@ -11,4 +11,6 @@ else
     with_go
 fi
 
-make local
+with_mage
+
+mage build:local

.buildkite/scripts/package.sh

@@ -35,10 +35,10 @@ with_mage
 case "${TYPE}" in
     "snapshot")
         export SNAPSHOT=true
-        make release
+        mage build:release
         ;;
     "staging")
-        make release
+        mage build:release
         ;;
     *)
     echo "The option is unsupported yet"