fix: package fleet-server using golang-crossbuild (#5392)

* fix: package fleet-server using golang-crossbuild and introduce packaging steps on every PR

* feat: save artifacts during packaging for DRA

* fix: disable CGO and buildmode pie

* doc: add changelog fragment

Author: pkoutsovasilis

.buildkite/pipeline.package.mbp.yml

@@ -26,6 +26,8 @@ steps:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
       machineType: "c2-standard-16"
+    artifact_paths:
+      - build/distributions/**
     plugins:
       - *oidc_plugin
 
@@ -38,6 +40,8 @@ steps:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
       machineType: "c2-standard-16"
+    artifact_paths:
+      - build/distributions/**
     plugins:
       - *oidc_plugin
 
@@ -51,6 +55,8 @@ steps:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
       machineType: "c2-standard-16"
+    artifact_paths:
+      - build/distributions/**
     plugins:
       - *oidc_plugin
 
@@ -65,6 +71,8 @@ steps:
       provider: "gcp"
       image: "${IMAGE_UBUNTU_X86_64}"
       machineType: "c2-standard-16"
+    artifact_paths:
+      - build/distributions/**
     plugins:
       - *oidc_plugin
 
@@ -76,6 +84,8 @@ steps:
       provider: "aws"
       imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
       instanceType: "t4g.2xlarge"
+    artifact_paths:
+      - build/distributions/**
     plugins:
       - *oidc_plugin
 
@@ -88,6 +98,8 @@ steps:
       provider: "aws"
       imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
       instanceType: "t4g.2xlarge"
+    artifact_paths:
+      - build/distributions/**
     plugins:
       - *oidc_plugin
 
@@ -101,6 +113,8 @@ steps:
       provider: "aws"
       imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
       instanceType: "t4g.2xlarge"
+    artifact_paths:
+      - build/distributions/**
     plugins:
       - *oidc_plugin
 
@@ -115,6 +129,8 @@ steps:
       provider: "aws"
       imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
       instanceType: "t4g.2xlarge"
+    artifact_paths:
+      - build/distributions/**
     plugins:
       - *oidc_plugin
 

.buildkite/pipeline.yml

@@ -3,6 +3,8 @@
 env:
   DOCKER_COMPOSE_VERSION: "1.25.5"
   TERRAFORM_VERSION: "1.6.4"
+  IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
+  IMAGE_UBUNTU_ARM_64: "core-ubuntu-2204-aarch64"
   IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-fleet-server-ubuntu-2204-fips"
 
 # This section is used to define the plugins that will be used in the pipeline.
@@ -22,6 +24,13 @@ common:
   - docker_elastic_login_plugin: &docker_elastic_login_plugin
       elastic/vault-docker-login#v0.6.0:
         secret_path: 'kv/ci-shared/platform-ingest/elastic_docker_registry'
+  # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/fleet-server/01-gcp-buildkite-oidc.tf
+  # This plugin authenticates to Google Cloud using the OIDC token.
+  - oidc_plugin: &oidc_plugin
+      elastic/oblt-google-auth#v1.2.0:
+        lifetime: 10800 # seconds
+        project-id: "elastic-observability-ci"
+        project-number: "911195782929"
 
 steps:
   - group: "Check and build"
@@ -33,18 +42,63 @@ steps:
         agents:
           provider: "gcp"
 
-      - label: ":building_construction: Run local build"
-        key: make-local
-        command: ".buildkite/scripts/local_build.sh"
+      - label: "Package x86_64"
+        key: "package-x86-64-pr"
+        env:
+          PLATFORMS: "linux/amd64,darwin/amd64,windows/amd64"
+        command: ".buildkite/scripts/release_test.sh"
+        artifact_paths:
+          - build/distributions/**
         agents:
           provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "c2-standard-16"
+        plugins:
+          - *oidc_plugin
 
-      - label: ":building_construction: Run local build: MacOS"
-        key: make-local-macos
-        command: ".buildkite/scripts/local_build.sh"
+      - label: "Package x86_64 FIPS"
+        key: "package-fips-x86-64-pr"
+        env:
+          FIPS: "true"
+          PLATFORMS: "linux/amd64"
+        command: ".buildkite/scripts/release_test.sh"
+        artifact_paths:
+          - build/distributions/**
         agents:
-          provider: orka
-          imagePrefix: generic-13-ventura-arm
+          provider: "gcp"
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "c2-standard-16"
+        plugins:
+          - *oidc_plugin
+
+      - label: "Package aarch64"
+        key: "package-arm64-pr"
+        env:
+          PLATFORMS: "linux/arm64,darwin/arm64"
+        command: ".buildkite/scripts/release_test.sh"
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "t4g.2xlarge"
+        plugins:
+          - *oidc_plugin
+
+      - label: "Package aarch64 FIPS"
+        key: "package-fips-arm64-pr"
+        env:
+          FIPS: "true"
+          PLATFORMS: "linux/arm64"
+        command: ".buildkite/scripts/release_test.sh"
+        artifact_paths:
+          - build/distributions/**
+        agents:
+          provider: "aws"
+          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          instanceType: "t4g.2xlarge"
+        plugins:
+          - *oidc_plugin
 
   - group: "Performance test"
     key: "performance-test"
@@ -295,22 +349,6 @@ steps:
     depends_on:
       - step: "publish"
 
-  - label: ":gcloud: Release test"
-    key: "release-test"
-    command: ".buildkite/scripts/release_test.sh"
-    agents:
-      provider: "gcp"
-    depends_on:
-      - step: "tests"
-        allow_failure: false
-    plugins:
-      # See https://github.com/elastic/oblt-infra/blob/main/conf/resources/repos/fleet-server/01-gcp-buildkite-oidc.tf
-      # This plugin authenticates to Google Cloud using the OIDC token.
-      - elastic/oblt-google-auth#v1.2.0:
-          lifetime: 10800 # seconds
-          project-id: "elastic-observability-ci"
-          project-number: "911195782929"
-
   - label: ":jenkins: Release - Package Registry Distribution"
     key: "release-package-registry"
     trigger: "package-registry-release-package-registry-distribution"

.buildkite/scripts/package.sh

@@ -35,10 +35,10 @@ with_mage
 case "${TYPE}" in
     "snapshot")
         export SNAPSHOT=true
-        mage build:release
+        mage docker:release
         ;;
     "staging")
-        mage build:release
+        mage docker:release
         ;;
     *)
     echo "The option is unsupported yet"

Dockerfile.fips

@@ -29,7 +29,7 @@ ARG SUFFIX
 FROM docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-${SUFFIX} AS builder
 
 WORKDIR /fleet-server
-
+ENV PATH="$PATH:/go/bin"
 COPY go.mod go.sum ./
 RUN go mod download && go mod verify
 RUN go install github.com/magefile/mage # uses version in go.mod

changelog/fragments/1756940128-fix-binary-compilation.yaml

@@ -0,0 +1,5 @@
+kind: bug-fix
+summary: Build fleet-server as fully static binary to restore OS matrix compatibility
+component: fleet-server
+pr: https://github.com/elastic/fleet-server/pull/5392
+issue: https://github.com/elastic/fleet-server/issues/5262

magefile.go

@@ -90,7 +90,6 @@ const (
 
 // const and vars used by magefile.
 const (
-	buildMode  = "pie"
 	binaryName = "fleet-server"
 	binaryExe  = "fleet-server.exe"
 
@@ -671,6 +670,7 @@ func (Check) All() {
 // FIPS creates a FIPS capable binary.
 func (Build) Local() error {
 	env := environMap()
+	env["CGO_ENABLED"] = "0"
 	if isFIPS() {
 		addFIPSEnvVars(env)
 	}
@@ -704,6 +704,7 @@ func goBuild(osArg, archArg string, cover bool) error {
 	env := environMap()
 	env["GOOS"] = osArg
 	env["GOARCH"] = archArg
+	env["CGO_ENABLED"] = "0"
 	distArr := []string{"fleet-server"}
 	if isFIPS() {
 		addFIPSEnvVars(env)
@@ -729,7 +730,6 @@ func goBuild(osArg, archArg string, cover bool) error {
 		"-tags=" + getTagsString(),
 		"-gcflags=" + getGCFlags(),
 		"-ldflags=" + getLDFlags(),
-		"-buildmode=" + buildMode,
 		"-o", outFile,
 	}
 	if cover {
@@ -1007,9 +1007,14 @@ func (Docker) Image() error {
 	} else if isDEV() {
 		version += "-dev"
 	}
+	suffix := dockerSuffix
+	if runtime.GOARCH == "arm64" {
+		suffix = dockerArmSuffix
+	}
 	if isFIPS() {
 		dockerFile = dockerBuilderFIPS
 		image += "-fips"
+		suffix += "-fips"
 	}
 	if v, ok := os.LookupEnv(envDockerImage); ok && v != "" {
 		image = v
@@ -1023,6 +1028,7 @@ func (Docker) Image() error {
 		"--build-arg", "VERSION="+getVersion(),
 		"--build-arg", "GCFLAGS="+getGCFlags(),
 		"--build-arg", "LDFLAGS="+getLDFlags(),
+		"--build-arg", "SUFFIX="+suffix,
 		"-f", dockerFile,
 		"-t", image+":"+version,
 		".",