Convert Makefile to magefile.go (#4912)

Reimplement Makefile targets in mage.
Change buildkite pipeline to use mage.
Change Dockerfile* to use mage.

(cherry picked from commit db5f46b)

# Conflicts:
#	.buildkite/pipeline.yml
#	.buildkite/scripts/common.sh
#	.buildkite/scripts/local_build.sh
#	.buildkite/scripts/test-release.sh
#	Dockerfile.build
#	Dockerfile.fips
#	Makefile
#	NOTICE.txt
#	README.md
#	dev-tools/cloud/Makefile
#	dev-tools/cloud/terraform/main.tf
#	dev-tools/e2e/Dockerfile
#	dev-tools/e2e/build.sh
#	dev-tools/e2e/validatecerts.go
#	dev-tools/go.mod
#	dev-tools/go.sum
#	docs/developers-guide.md
#	docs/docker-images.md
#	docs/fips.md
#	testing/e2e/fips_test.go

Author: michel-laterman

.buildkite/pipeline.yml

@@ -2,12 +2,6 @@
 
 env:
   DOCKER_COMPOSE_VERSION: "1.25.5"
-  DOCKER_REGISTRY: "docker.elastic.co"
-  DOCKER_IMAGE: "${DOCKER_REGISTRY}/observability-ci/fleet-server" # needs to rename for rollback
-  DOCKER_IMAGE_SHA_TAG: "git-${BUILDKITE_COMMIT:0:12}" # needs to rename for rollback, should be "git-${BUILDKITE_COMMIT:0:12}"
-  DOCKER_IMAGE_LATEST_TAG: "latest" # needs to rename for rollback
-  DOCKER_IMAGE_GIT_TAG: "${BUILDKITE_BRANCH}" # needs to rename for rollback
-  GO_AGENT_IMAGE: "golang:${GO_VERSION}"
   TERRAFORM_VERSION: "1.6.4"
 
 steps:
@@ -16,19 +10,15 @@ steps:
     steps:
       - label: ":white_check_mark: Run check-ci"
         key: check-ci
-        command: "make check-ci"
+        command: ".buildkite/scripts/check_ci.sh"
         agents:
-          image: "${GO_AGENT_IMAGE}"
-          cpu: "8"
-          memory: "4G"
+          provider: "gcp"
 
       - label: ":building_construction: Run local build"
         key: make-local
-        command: "make local"
+        command: ".buildkite/scripts/local_build.sh"
         agents:
-          image: "${GO_AGENT_IMAGE}"
-          cpu: "8"
-          memory: "4G"
+          provider: "gcp"
 
       - label: ":building_construction: Run local build: MacOS"
         key: make-local-macos
@@ -43,6 +33,8 @@ steps:
       - label: "Run go benchmark for PR branch"
         key: "go-benchmark-pr"
         command: ".buildkite/scripts/run_benchmark.sh pr"
+        env:
+          BENCHMARK_ARGS: "-count=8 -benchmem"
         artifact_paths:
           - build/next.out
           - build/next.stat
@@ -51,8 +43,11 @@ steps:
           machineType: "c2-standard-8"
 
       - label: "Run go benchmark for ${BUILDKITE_PULL_REQUEST_BASE_BRANCH}"
+        skip: true # FIXME: Temporarily disabled until mage is on main
         key: "go-benchmark-base"
         command: ".buildkite/scripts/run_benchmark.sh base"
+        env:
+          BENCHMARK_ARGS: "-count=8 -benchmem"
         artifact_paths:
           - build/base.out
           - build/base.stat
@@ -61,6 +56,7 @@ steps:
           machineType: "c2-standard-8"
 
       - label: "Compare results"
+        skip: true # FIXME: Temporarily disabled until mage is on main
         key: "go-benchmark-compare"
         command: ".buildkite/scripts/run_benchmark.sh compare"
         artifact_paths:
@@ -87,6 +83,31 @@ steps:
           - build/*.xml
           - build/coverage*.out
 
+<<<<<<< HEAD
+=======
+      - label: ":smartbear-testexecute: Run unit tests with requirefips build tag"
+        key: unit-test-fips-tag
+        command: ".buildkite/scripts/unit_test.sh"
+        env:
+          FIPS: "true"
+        agents:
+          provider: "gcp"
+        artifact_paths:
+          - build/*.xml
+          - build/coverage*.out
+
+      - label: ":smartbear-testexecute: Run fips140=only unit tests"
+        key: unit-test-fips140-only
+        command: ".buildkite/scripts/unit_test_fipsonly.sh"
+        env:
+          FIPS: "true"
+        agents:
+          provider: "gcp"
+        artifact_paths:
+          - build/*.xml
+          - build/coverage*.out
+
+>>>>>>> db5f46b (Convert Makefile to magefile.go (#4912))
       - label: ":smartbear-testexecute: Run unit tests: MacOS 13"
         key: unit-test-macos-13
         command: ".buildkite/scripts/unit_test.sh"
@@ -111,13 +132,25 @@ steps:
       - label: "E2E Test"
         key: "e2e-test"
         command: ".buildkite/scripts/e2e_test.sh"
+<<<<<<< HEAD
         env:
           TESTCONTAINERS_RYUK_CONTAINER_PRIVILEGED: "true"
+=======
+>>>>>>> db5f46b (Convert Makefile to magefile.go (#4912))
         agents:
           provider: "gcp"
         artifact_paths:
           - build/*.xml
           - build/e2e-coverage.out
+<<<<<<< HEAD
+=======
+        plugins:
+          - test-collector#v1.10.2:
+              files: "build/test-*.xml"
+              format: "junit"
+              branches: "main"
+              debug: true
+>>>>>>> db5f46b (Convert Makefile to magefile.go (#4912))
 
       - label: ":junit: Junit annotate"
         plugins:
@@ -154,6 +187,13 @@ steps:
 
       - label: ":gcloud: Cloud e2e Test"
         key: "cloud-e2e-test"
+        env:
+          DOCKER_REGISTRY: "docker.elastic.co"
+          DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-cloud-fleet"
+          DOCKER_IMAGE_TAG: "pr-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_COMMIT:0:12}"
+          SNAPSHOT: "true"
+          PLATFORMS: "linux/amd64"
+          TF_VAR_pull_request: "${BUILDKITE_PULL_REQUEST}"
         command: ".buildkite/scripts/cloud_e2e_test.sh"
         agents:
           provider: "gcp"
@@ -169,6 +209,12 @@ steps:
   - label: ":docker: Publish docker image"
     key: "publish"
     command: ".buildkite/scripts/build_push_docker_image.sh"
+    env:
+      DOCKER_REGISTRY: "docker.elastic.co"
+      DOCKER_IMAGE: "docker.elastic.co/observability-ci/fleet-server" # needs to rename for rollback
+      DOCKER_IMAGE_SHA_TAG: "git-${BUILDKITE_COMMIT:0:12}" # needs to rename for rollback, should be "git-${BUILDKITE_COMMIT:0:12}"
+      DOCKER_IMAGE_LATEST_TAG: "latest" # needs to rename for rollback
+      DOCKER_IMAGE_GIT_TAG: "${BUILDKITE_BRANCH}" # needs to rename for rollback
     if: "build.env('BUILDKITE_PULL_REQUEST') == 'false' && build.env('BUILDKITE_BRANCH') == 'main'"
     agents:
       provider: "gcp"

.buildkite/scripts/build_push_docker_image.sh

@@ -4,14 +4,20 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
+add_bin_path
+
+with_go
+
+with_mage
+
 echo "Building the docker image..."
 if ! docker pull -q ${DOCKER_IMAGE}:${DOCKER_IMAGE_SHA_TAG} 2> /dev/null; then
     DOCKER_IMAGE_TAG="${DOCKER_IMAGE_SHA_TAG}"
-    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG} make build-and-push-docker
+    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG=${DOCKER_IMAGE_TAG} mage docker:image docker:push
 fi
 
 if [[ "${DOCKER_IMAGE_GIT_TAG}" == "main" ]]; then
-    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG="${DOCKER_IMAGE_LATEST_TAG}" make build-and-push-docker
+    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG="${DOCKER_IMAGE_LATEST_TAG}" mage docker:image docker:push
 elif [[ ${BUILDKITE_PULL_REQUEST} == "false" ]]; then
-    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG="${DOCKER_IMAGE_GIT_TAG}" make build-and-push-docker
+    DOCKER_IMAGE=${DOCKER_IMAGE} DOCKER_IMAGE_TAG="${DOCKER_IMAGE_GIT_TAG}" mage docker:image docker:push
 fi

.buildkite/scripts/check_ci.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common.sh
+
+add_bin_path
+
+with_go
+
+with_mage
+
+mage check:ci

.buildkite/scripts/cloud_e2e_test.sh

@@ -12,4 +12,37 @@ with_Terraform
 
 with_docker_compose
 
-USER=fleetserverci make test-cloude2e
+with_mage
+
+cleanup() {
+  r=$?
+
+  if [ -f dev-tools/cloud/terraform/.terraform.lock.hcl ] ; then
+    echo "--- Deployment detected, running cleanup."
+    mage test:cloudE2EDown
+  else
+      echo "--- No deployment detected, skipping cleanup."
+  fi
+  exit $r
+}
+trap cleanup EXIT INT TERM
+
+USER=fleetserverci mage docker:cover docker:customAgentImage docker:push test:cloudE2EUp
+FLEET_SERVER_URL=$(terraform output --raw --state=dev-tools/cloud/terraform/terraform.tfstate fleet_url)
+echo "Fleet server: \"${FLEET_SERVER_URL}\""
+echo "Deployment ID: $(terraform output --raw --state=dev-tools/cloud/terraform/terraform.tfstate deployment_id)"
+
+if [[ "${FLEET_SERVER_URL}" == "" ]]; then
+    message="FLEET_SERVER_URL is empty, cloud e2e tests cannot be executed"
+    if [[ "${CI}" == "true" ]]; then
+        buildkite-agent annotate \
+            "${message}" \
+            --context "ctx-cloude2e-test" \
+            --style "error"
+    fi
+    echo "${message}"
+    exit 0
+fi
+
+echo "--- Trigger cloud E2E test"
+mage test:cloudE2ERun

.buildkite/scripts/common.sh

@@ -51,6 +51,21 @@ with_go() {
     export PATH="${PATH}:$(go env GOPATH):$(go env GOPATH)/bin"
 }
 
+<<<<<<< HEAD
+=======
+with_msft_go() {
+    echo "Setting up microsoft/go"
+    create_workspace
+    check_platform_architeture
+    MSFT_DOWNLOAD_URL=https://aka.ms/golang/release/latest/go$(cat .go-version).${platform_type}-${arch_type}.tar.gz
+    retry 5 $(curl -sL -o - $MSFT_DOWNLOAD_URL | tar -xz -f - -C ${WORKSPACE})
+    export PATH="${PATH}:${WORKSPACE}/go/bin"
+    go version
+    which go
+    export PATH="${PATH}:$(go env GOPATH)/bin"
+}
+
+>>>>>>> db5f46b (Convert Makefile to magefile.go (#4912))
 with_docker_compose() {
     echo "Setting up the Docker-compose environment..."
     create_workspace
@@ -136,17 +151,11 @@ download_mbp_packages_from_gcp_bucket() {
 }
 
 with_mage() {
-    local install_packages=(
-            "github.com/magefile/mage"
-            "github.com/elastic/go-licenser"
-            "golang.org/x/tools/cmd/goimports"
-            "github.com/jstemmer/go-junit-report"
-            "gotest.tools/gotestsum"
-    )
     create_workspace
-    for pkg in "${install_packages[@]}"; do
-        go install "${pkg}@latest"
-    done
+    go install github.com/magefile/mage # uses go.mod implicitly
+    mage -clean
+    mage -version
+    which mage
 }
 
 cleanup() {

.buildkite/scripts/dra_release.sh

@@ -4,6 +4,12 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
+add_bin_path
+
+with_go
+
+with_mage
+
 FOLDER_PATH="build/distributions"
 BASE_DIR="${WORKSPACE}/${FOLDER_PATH}"
 DRA_OUTPUT="release-manager.out"
@@ -12,7 +18,7 @@ export TYPE=${1}
 # DRA_BRANCH can be used for manually testing packaging with PRs
 # e.g. define `DRA_BRANCH="main"` under Options/Environment Variables in the Buildkite UI after clicking new Build
 export BRANCH="${DRA_BRANCH:="${BUILDKITE_BRANCH:=""}"}"
-export VERSION="$(make get-version)"
+export VERSION="$(mage getVersion)"
 
 if [[ "${VERSION}" == *"-SNAPSHOT"* || "${VERSION}" == "" ]]; then
     echo "The 'version' parameter is required and it cannot contain the suffix '-SNAPSHOT'."
@@ -24,14 +30,7 @@ if [[ "${PROJECT}" == "" ]]; then
     exit 1
 fi
 
-add_bin_path
-
 download_mbp_packages_from_gcp_bucket "${FOLDER_PATH}" "${TYPE}"
-
-with_go
-
-with_mage
-
 export RM_VERSION="${VERSION}"
 
 if [[ ${TYPE} == "snapshot" ]]; then

.buildkite/scripts/e2e_test.sh

@@ -10,5 +10,7 @@ with_go
 
 with_docker_compose
 
+with_mage
+
 echo "Starting the E2E tests..."
-make test-e2e junit-report
+mage test:e2e test:junitReport

.buildkite/scripts/integration_test.sh

@@ -10,5 +10,7 @@ with_go
 
 with_docker_compose
 
+with_mage
+
 echo "Starting the integration tests..."
-make test-int junit-report
+mage test:integration test:junitReport

.buildkite/scripts/local_build.sh

@@ -7,4 +7,10 @@ source .buildkite/scripts/common.sh
 add_bin_path
 with_go
 
-make local
+<<<<<<< HEAD
+make local
+=======
+with_mage
+
+mage build:local
+>>>>>>> db5f46b (Convert Makefile to magefile.go (#4912))

.buildkite/scripts/package.sh

@@ -27,10 +27,10 @@ with_mage
 case "${TYPE}" in
     "snapshot")
         export SNAPSHOT=true
-        make release
+        mage build:release
         ;;
     "staging")
-        make release
+        mage build:release
         ;;
     *)
     echo "The option is unsupported yet"