[9.0](backport #5392) fix: package fleet-server using golang-crossbuild (#5429)

* fix: package fleet-server using golang-crossbuild  (#5392)

* fix: package fleet-server using golang-crossbuild and introduce packaging steps on every PR

* feat: save artifacts during packaging for DRA

* fix: disable CGO and buildmode pie

* doc: add changelog fragment

(cherry picked from commit 7da3f69)

# Conflicts:
#	.buildkite/pipeline.package.mbp.yml
#	.buildkite/pipeline.yml
#	.buildkite/scripts/package.sh
#	Dockerfile.fips
#	magefile.go

* fix: adjust changes for make

* fix: fleet-server-package-mbp CI step dependencies (#5446)

(cherry picked from commit 5aa10ef)

---------

Co-authored-by: Panos Koutsovasilis <[email protected]>

Author: mergify[bot]

.buildkite/pipeline.package.mbp.yml

@@ -28,6 +28,8 @@ steps:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
       machineType: "c2-standard-16"
+    artifact_paths:
+      - build/distributions/**
     plugins:
       - *oidc_plugin
 
@@ -40,6 +42,8 @@ steps:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
       machineType: "c2-standard-16"
+    artifact_paths:
+      - build/distributions/**
     plugins:
       - *oidc_plugin
 
@@ -51,6 +55,8 @@ steps:
       provider: "aws"
       imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
       instanceType: "t4g.2xlarge"
+    artifact_paths:
+      - build/distributions/**
     plugins:
       - *oidc_plugin
 
@@ -63,6 +69,8 @@ steps:
       provider: "aws"
       imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
       instanceType: "t4g.2xlarge"
+    artifact_paths:
+      - build/distributions/**
     plugins:
       - *oidc_plugin
 

.buildkite/pipeline.yml

@@ -9,6 +9,19 @@ env:
   DOCKER_IMAGE_GIT_TAG: "${BUILDKITE_BRANCH}" # needs to rename for rollback
   GO_AGENT_IMAGE: "golang:${GO_VERSION}"
   TERRAFORM_VERSION: "1.6.4"
+  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2204-aarch64"
+
+# This section is used to define the plugins that will be used in the pipeline.
+# See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
+common:
+  # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/fleet-server/01-gcp-buildkite-oidc.tf
+  # This plugin authenticates to Google Cloud using the OIDC token.
+  - oidc_plugin: &oidc_plugin
+      elastic/oblt-google-auth#v1.2.0:
+        lifetime: 10800 # seconds
+        project-id: "elastic-observability-ci"
+        project-number: "911195782929"
 
 steps:
   - group: "Check and build"
@@ -22,20 +35,33 @@ steps:
           cpu: "8"
           memory: "4G"
 
-      - label: ":building_construction: Run local build"
-        key: make-local
-        command: "make local"
+      - label: "Package x86_64"
+        key: "package-x86-64-pr"
+        env:
+          PLATFORMS: "linux/amd64 darwin/amd64 windows/amd64"
+        command: ".buildkite/scripts/release_test.sh"
+        artifact_paths:
+          - build/distributions/**
         agents:
-          image: "${GO_AGENT_IMAGE}"
-          cpu: "8"
-          memory: "4G"
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "c2-standard-16"
+        plugins:
+          - *oidc_plugin
 
-      - label: ":building_construction: Run local build: MacOS"
-        key: make-local-macos
-        command: ".buildkite/scripts/local_build.sh"
+      - label: "Package aarch64"
+        key: "package-arm64-pr"
+        env:
+          PLATFORMS: "linux/arm64 darwin/arm64"
+        command: ".buildkite/scripts/release_test.sh"
+        artifact_paths:
+          - build/distributions/**
         agents:
-          provider: orka
-          imagePrefix: generic-13-ventura-arm
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "t4g.2xlarge"
+        plugins:
+          - *oidc_plugin
 
   - group: "Performance test"
     key: "performance-test"
@@ -210,22 +236,6 @@ steps:
     depends_on:
       - step: "publish"
 
-  - label: ":gcloud: Release test"
-    key: "release-test"
-    command: ".buildkite/scripts/release_test.sh"
-    agents:
-      provider: "gcp"
-    depends_on:
-      - step: "tests"
-        allow_failure: false
-    plugins:
-      # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/fleet-server/01-gcp-buildkite-oidc.tf
-      # This plugin authenticates to Google Cloud using the OIDC token.
-      - elastic/oblt-google-auth#v1.2.0:
-          lifetime: 10800 # seconds
-          project-id: "elastic-observability-ci"
-          project-number: "911195782929"
-
   - label: ":jenkins: Release - Package Registry Distribution"
     key: "release-package-registry"
     trigger: "package-registry-release-package-registry-distribution"
@@ -244,5 +254,7 @@ steps:
     build:
       branch: "${BUILDKITE_BRANCH}"
     depends_on:
-      - step: "release-test"
+      - step: "package-x86-64-pr"
+        allow_failure: false
+      - step: "package-arm64-pr"
         allow_failure: false

.buildkite/scripts/package.sh

@@ -27,10 +27,10 @@ with_mage
 case "${TYPE}" in
     "snapshot")
         export SNAPSHOT=true
-        make release
+        make docker-release
         ;;
     "staging")
-        make release
+        make docker-release
         ;;
     *)
     echo "The option is unsupported yet"

.buildkite/scripts/test-release.sh

@@ -3,11 +3,26 @@
 set -euo pipefail
 
 FLEET_SERVER_VERSION=${1:?"Fleet Server version is needed"}
+PLATFORMS=${PLATFORMS:-"darwin/amd64 darwin/arm64 linux/amd64 linux/arm64 windows/amd64"}
 
-PLATFORM_FILES=(darwin-aarch64.tar.gz darwin-x86_64.tar.gz linux-arm64.tar.gz linux-x86_64.tar.gz windows-x86_64.zip)
-if [ "$FIPS" = "true" ] ; then
-    PLATFORM_FILES=(linux-arm64-fips.tar.gz linux-x86_64-fips.tar.gz)
-fi
+PLATFORM_FILES=()
+for p in $PLATFORMS; do
+  os="${p%%/*}"
+  arch="${p##*/}"
+
+  case "$os/$arch" in
+    darwin/arm64) arch="aarch64" ;;
+    */amd64) arch="x86_64" ;;
+  esac
+
+  case "$os" in
+    windows) ext="zip" ;;
+    *) ext="tar.gz" ;;
+  esac
+
+  file="${os}-${arch}.${ext}"
+  PLATFORM_FILES+=("$file")
+done
 
 #make release
 
@@ -23,17 +38,17 @@ for PLATFORM_FILE in "${PLATFORM_FILES[@]}"
 do
     file="${FILE_PREFIX}${PLATFORM_FILE}"
     if [ ! -f "${file}" ]; then
-        echo -e "${RED}!! ${PLATFORM_FILE}: The file was not created.${NO_COLOR}"
+        echo -e "${RED}!! ${file}: The file was not created.${NO_COLOR}"
         exit 1
     else
-        echo -e "- ${PLATFORM_FILE} ${GREEN}OK${NO_COLOR}"
+        echo -e "- ${file} ${GREEN}OK${NO_COLOR}"
     fi
 
     fileSha512="${file}.sha512"
     if [ ! -f "${fileSha512}" ]; then
         echo -e "${RED}!! ${fileSha512}: The file was not created.${NO_COLOR}"
         exit 1
     else
-        echo -e "- ${PLATFORM_FILE}.sha512 ${GREEN}OK${NO_COLOR}"
+        echo -e "- ${file}.sha512 ${GREEN}OK${NO_COLOR}"
     fi
 done

Makefile

@@ -12,11 +12,6 @@ DEFAULT_VERSION=$(shell awk '/const DefaultVersion/{print $$NF}' version/version
 TARGET_ARCH_amd64=x86_64
 TARGET_ARCH_arm64=arm64
 PLATFORMS ?= darwin/amd64 darwin/arm64 linux/amd64 linux/arm64 windows/amd64
-BUILDMODE_linux_amd64=-buildmode=pie
-BUILDMODE_linux_arm64=-buildmode=pie
-BUILDMODE_windows_amd64=-buildmode=pie
-BUILDMODE_darwin_amd64=-buildmode=pie
-BUILDMODE_darwin_arm64=-buildmode=pie
 
 CROSSBUILD_SUFFIX=main-debian11
 CROSSBUILD_ARM_SUFFIX=base-arm-debian9
@@ -65,6 +60,10 @@ ifeq "${FIPS}" "true"
 DOCKER_IMAGE_TAG:=${DOCKER_IMAGE_TAG}-fips
 endif
 endif
+CGO_ENABLED?=0
+ifeq "${FIPS}" "true"
+CGO_ENABLED=1
+endif
 DOCKER_IMAGE?=docker.elastic.co/fleet-server/fleet-server
 
 PLATFORM_TARGETS=$(addprefix release-, $(PLATFORMS))
@@ -138,7 +137,7 @@ list-platforms: ## - Show the possible PLATFORMS
 .PHONY: local
 local: ## - Build local binary for local environment (bin/fleet-server)
 	@printf "${CMD_COLOR_ON} Build binaries using local go installation\n${CMD_COLOR_OFF}"
-	${GOFIPSEXPERIMENT} go build -tags=${GOBUILDTAGS} -gcflags="${GCFLAGS}" -ldflags="${LDFLAGS}" -o ./bin/fleet-server .
+	CGO_ENABLED=${CGO_ENABLED} ${GOFIPSEXPERIMENT} go build -tags=${GOBUILDTAGS} -gcflags="${GCFLAGS}" -ldflags="${LDFLAGS}" -o ./bin/fleet-server .
 	@printf "${CMD_COLOR_ON} Binaries in ./bin/\n${CMD_COLOR_OFF}"
 
 .PHONY: $(COVER_TARGETS)
@@ -147,8 +146,7 @@ $(COVER_TARGETS): cover-%: ## - Build a binary with the -cover flag for integrat
 	$(eval $@_OS := $(firstword $(subst /, ,$(lastword $(subst cover-, ,$@)))))
 	$(eval $@_GO_ARCH := $(lastword $(subst /, ,$(lastword $(subst cover-, ,$@)))))
 	$(eval $@_ARCH := $(TARGET_ARCH_$($@_GO_ARCH)))
-	$(eval $@_BUILDMODE:= $(BUILDMODE_$($@_OS)_$($@_GO_ARCH)))
-	GOOS=$($@_OS) GOARCH=$($@_GO_ARCH) ${GOFIPSEXPERIMENT} go build -tags=${GOBUILDTAGS} -cover -coverpkg=./... -gcflags="${GCFLAGS}" -ldflags="${LDFLAGS}" $($@_BUILDMODE) -o build/cover/fleet-server-$(VERSION)-$($@_OS)-$($@_ARCH)$(FIPSSUFFIX)/fleet-server$(if $(filter windows,$($@_OS)),.exe,) .
+	CGO_ENABLED=${CGO_ENABLED} GOOS=$($@_OS) GOARCH=$($@_GO_ARCH) ${GOFIPSEXPERIMENT} go build -tags=${GOBUILDTAGS} -cover -coverpkg=./... -gcflags="${GCFLAGS}" -ldflags="${LDFLAGS}" -o build/cover/fleet-server-$(VERSION)-$($@_OS)-$($@_ARCH)$(FIPSSUFFIX)/fleet-server$(if $(filter windows,$($@_OS)),.exe,) .
 
 .PHONY: clean
 clean: ## - Clean up build artifacts
@@ -224,17 +222,17 @@ test-release:  ## - Check that all release binaries are created
 # If FIPS=true unit tests need microsoft/go + OpenSSL with FIPS
 .PHONY: test-unit
 test-unit: prepare-test-context  ## - Run unit tests only
-	set -o pipefail; ${GOFIPSEXPERIMENT} go test ${GO_TEST_FLAG} -tags=$(GOBUILDTAGS) -v -race -coverprofile=build/coverage-${OS_NAME}.out ./... | tee build/test-unit-${OS_NAME}.out
+	set -o pipefail; ${GOFIPSEXPERIMENT} CGO_ENABLED=1 go test ${GO_TEST_FLAG} -tags=$(GOBUILDTAGS) -v -race -coverprofile=build/coverage-${OS_NAME}.out ./... | tee build/test-unit-${OS_NAME}.out
 
 # FIPS unit tests are meant to use go v1.24 to check FIPS compliance.
 # This check is very strict, and should be thought of as a static-code analysis tool.
 .PHONY: test-unit-fips
 test-unit-fips: prepare-test-context  ## - Run unit tests with go 1.24's fips140=only for testing
-	set -o pipefail; GOFIPS140=latest GODEBUG=fips140=only go test ${GO_TEST_FLAG} -tags=$(GOBUILDTAGS) -v -race -coverprofile=build/coverage-${OS_NAME}.out ./... | tee build/test-unit-fips-${OS_NAME}.out
+	set -o pipefail; GOFIPS140=latest GODEBUG=fips140=only CGO_ENABLED=1 go test ${GO_TEST_FLAG} -tags=$(GOBUILDTAGS) -v -race -coverprofile=build/coverage-${OS_NAME}.out ./... | tee build/test-unit-fips-${OS_NAME}.out
 
 .PHONY: benchmark
 benchmark: prepare-test-context install-benchstat  ## - Run benchmark tests only
-	set -o pipefail; go test -bench=$(BENCHMARK_FILTER) -tags=$(GOBUILDTAGS) -run=$(BENCHMARK_FILTER) $(BENCHMARK_ARGS) $(BENCHMARK_PACKAGE) | tee "build/$(BENCH_BASE)"
+	set -o pipefail; CGO_ENABLED=1 go test -bench=$(BENCHMARK_FILTER) -tags=$(GOBUILDTAGS) -run=$(BENCHMARK_FILTER) $(BENCHMARK_ARGS) $(BENCHMARK_PACKAGE) | tee "build/$(BENCH_BASE)"
 
 .PHONY: install-benchstat
 install-benchstat: ## - Install the benchstat package
@@ -269,8 +267,7 @@ $(PLATFORM_TARGETS): release-%:
 	$(eval $@_OS := $(firstword $(subst /, ,$(lastword $(subst release-, ,$@)))))
 	$(eval $@_GO_ARCH := $(lastword $(subst /, ,$(lastword $(subst release-, ,$@)))))
 	$(eval $@_ARCH := $(TARGET_ARCH_$($@_GO_ARCH)))
-	$(eval $@_BUILDMODE:= $(BUILDMODE_$($@_OS)_$($@_GO_ARCH)))
-	GOOS=$($@_OS) GOARCH=$($@_GO_ARCH) ${GOFIPSEXPERIMENT} go build -tags=${GOBUILDTAGS} -gcflags="${GCFLAGS}" -ldflags="${LDFLAGS}" $($@_BUILDMODE) -o build/binaries/fleet-server-$(VERSION)-$($@_OS)-$($@_ARCH)$(FIPSSUFFIX)/fleet-server .
+	CGO_ENABLED=${CGO_ENABLED} GOOS=$($@_OS) GOARCH=$($@_GO_ARCH) ${GOFIPSEXPERIMENT} go build -tags=${GOBUILDTAGS} -gcflags="${GCFLAGS}" -ldflags="${LDFLAGS}" -o build/binaries/fleet-server-$(VERSION)-$($@_OS)-$($@_ARCH)$(FIPSSUFFIX)/fleet-server .
 	@$(MAKE) OS=$($@_OS) ARCH=$($@_ARCH) package-target
 
 .PHONY: build-docker
@@ -321,6 +318,8 @@ endif
 build-releaser: ## - Build a Docker image to run make package including all build tools
 ifeq ($(shell uname -p),arm)
 	$(eval SUFFIX := ${CROSSBUILD_ARM_SUFFIX})
+else ifeq ($(shell uname -p),aarch64)
+	$(eval SUFFIX := ${CROSSBUILD_ARM_SUFFIX})
 else
 	$(eval SUFFIX := ${CROSSBUILD_SUFFIX})
 endif
@@ -332,16 +331,16 @@ endif
 
 .PHONY: docker-release
 docker-release: build-releaser ## - Builds a release for all platforms in a dockerised environment
-	docker run --rm -u $(shell id -u):$(shell id -g) --env=GOCACHE=/go/cache --volume $(PWD):/go/src/github.com/elastic/fleet-server $(BUILDER_IMAGE) release
+	docker run --rm -u $(shell id -u):$(shell id -g) --env=GOCACHE=/go/cache --env='PLATFORMS=${PLATFORMS}' --volume $(PWD):/go/src/github.com/elastic/fleet-server $(BUILDER_IMAGE) release
 
 .PHONY: docker-cover-e2e-binaries
 docker-cover-e2e-binaries: build-releaser
 ifeq "${FIPS}" "true"
 	## non-linux is currently unsupported for FIPS
-	docker run --rm -u $(shell id -u):$(shell id -g) --env=GOCACHE=/go/cache --volume $(PWD):/go/src/github.com/elastic/fleet-server -e SNAPSHOT=true -e DEV=$(DEV) -e FIPS=$(FIPS) $(BUILDER_IMAGE) cover-linux/$(shell go env GOARCH)
+	docker run --rm -u $(shell id -u):$(shell id -g) --env=GOCACHE=/go/cache --env='PLATFORMS=${PLATFORMS}' --volume $(PWD):/go/src/github.com/elastic/fleet-server -e SNAPSHOT=true -e DEV=$(DEV) -e FIPS=$(FIPS) $(BUILDER_IMAGE) cover-linux/$(shell go env GOARCH)
 else
 	## Build for local architecture and for linux/$ARCH for docker images.
-	docker run --rm -u $(shell id -u):$(shell id -g) --env=GOCACHE=/go/cache --volume $(PWD):/go/src/github.com/elastic/fleet-server -e SNAPSHOT=true -e DEV=$(DEV) -e FIPS=$(FIPS) $(BUILDER_IMAGE) cover-linux/$(shell go env GOARCH) cover-$(shell go env GOOS)/$(shell go env GOARCH)
+	docker run --rm -u $(shell id -u):$(shell id -g) --env=GOCACHE=/go/cache --env='PLATFORMS=${PLATFORMS}' --volume $(PWD):/go/src/github.com/elastic/fleet-server -e SNAPSHOT=true -e DEV=$(DEV) -e FIPS=$(FIPS) $(BUILDER_IMAGE) cover-linux/$(shell go env GOARCH) cover-$(shell go env GOOS)/$(shell go env GOARCH)
 endif
 
 .PHONY: release
@@ -411,6 +410,7 @@ test-int-set: ## - Run integration tests without setup
 	REMOTE_ELASTICSEARCH_SERVICE_TOKEN=$(shell ./dev-tools/integration/get-elasticsearch-servicetoken.sh https://${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}@${TEST_REMOTE_ELASTICSEARCH_HOST} "fleet-server-remote") \
 	REMOTE_ELASTICSEARCH_CA_CRT_BASE64="$(shell COMPOSE_PROJECT_NAME=integration docker compose  -f ./dev-tools/e2e/docker-compose.yml --env-file ./dev-tools/integration/.env exec elasticsearch-remote /bin/bash -c "cat /usr/share/elasticsearch/config/certs/ca/ca.crt" | base64)" \
 	ELASTICSEARCH_HOSTS=${TEST_ELASTICSEARCH_HOSTS} ELASTICSEARCH_USERNAME=${ELASTICSEARCH_USERNAME} ELASTICSEARCH_PASSWORD=${ELASTICSEARCH_PASSWORD} \
+	CGO_ENABLED=1 \
 	go test -v -tags=integration -count=1 -race -p 1 ./...
 
 ##################################################
@@ -472,4 +472,4 @@ test-cloude2e: prepare-test-context  ## - Run cloude2e tests with full setup (sl
 test-cloude2e-set: ## Run cloude2e test
 	$(eval FLEET_SERVER_URL := $(shell make --no-print-directory -C ${CLOUD_TESTING_BASE} cloud-get-fleet-url))
 	make -C ${CLOUD_TESTING_BASE} cloud-get-fleet-url
-	FLEET_SERVER_URL="${FLEET_SERVER_URL}" go test -v -tags=cloude2e -count=1 -race -p 1 ./testing/cloude2e
+	FLEET_SERVER_URL="${FLEET_SERVER_URL}" CGO_ENABLED=1 go test -v -tags=cloude2e -count=1 -race -p 1 ./testing/cloude2e

changelog/fragments/1756940128-fix-binary-compilation.yaml

@@ -0,0 +1,5 @@
+kind: bug-fix
+summary: Build fleet-server as fully static binary to restore OS matrix compatibility
+component: fleet-server
+pr: https://github.com/elastic/fleet-server/pull/5392
+issue: https://github.com/elastic/fleet-server/issues/5262