Add Pipeline to deploy custom agent image for FIPS testing (#8035)

michel-laterman · mergify[bot] · commit 7738141ae23a · 2025-06-09T19:44:05.000Z
Add a new buildkite pipeline to build a custom agent image and use it in an ECH deployment for testing. Run FIPS integration tests on VMs with a FIPS provider. (cherry picked from commit 5908ed2) # Conflicts: # testing/integration/fleetserver_fips_test.go
diff --git a/.buildkite/bk.integration-fips.pipeline.yml b/.buildkite/bk.integration-fips.pipeline.yml
@@ -0,0 +1,143 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  DOCKER_REGISTRY: "docker.elastic.co"
+  ASDF_MAGE_VERSION: 1.14.0
+
+  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1749258065"
+  IMAGE_UBUNTU_X86_64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-1748955449"
+  IMAGE_UBUNTU_ARM64_FIPS: "platform-ingest-elastic-agent-ubuntu-2204-fips-aarch64-1748955449"
+
+steps:
+  - label: Build and push custom elastic-agent image
+    depends_on:
+      - 'packaging-containers-x86-64-fips' # Reuse artifacts produced in .buildkite/integration.pipeline.yml
+    key: integration-fips-cloud-image
+    env:
+      FIPS: "true"
+      CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
+      CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud"
+      TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud:git-${BUILDKITE_COMMIT:0:12}"
+    command: |
+      buildkite-agent artifact download build/distributions/elastic-agent-fips-cloud-*-linux-amd64.docker.tar.gz . --step 'packaging-containers-x86-64-fips'
+      mage cloud:load
+      mage cloud:push
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+      image: "${IMAGE_UBUNTU_2404_X86_64}"
+
+  - label: Start ESS stack for FIPS integration tests
+    key: integration-fips-ess
+    depends_on:
+      - integration-fips-cloud-image
+    env:
+      ASDF_TERRAFORM_VERSION: 1.9.2
+      CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
+      CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud"
+      TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud:git-${BUILDKITE_COMMIT:0:12}"
+    command: |
+      source .buildkite/scripts/steps/ess_start.sh
+    artifact_paths:
+      - test_infra/ess/*.tfstate
+      - test_infra/ess/*.lock.hcl
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true
+
+  - group: "fips:Stateful:Ubuntu"
+    key: integration-tests-ubuntu-fips
+    depends_on:
+      - integration-fips-ess
+    steps:
+      - label: "fips:x86_64:sudo-{{matrix.sudo}}:{{matrix.groups}}"
+        depends_on:
+          - packaging-ubuntu-x86-64-fips # Reuse artifacts produced in .buildkite/integration.pipeline.yml
+        env:
+          FIPS: "true"
+          CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
+          CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud"
+          TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud:git-${BUILDKITE_COMMIT:0:12}"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64-fips'
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix.groups}} {{matrix.sudo}}
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        retry:
+          automatic:
+            limit: 1
+        agents:
+          provider: "aws"
+          image: "${IMAGE_UBUNTU_X86_64_FIPS}"
+          instanceType: "m5.2xlarge"
+        matrix:
+          setup:
+            sudo:
+              - "false"
+              - "true"
+            groups:
+              - fleet # currently there is only a single test in the fleet group, add more tests once they have been defined
+
+      - label: "fips:arm64:sudo-{{matrix.sudo}}:{{matrix.groups}}"
+        depends_on:
+          - packaging-ubuntu-arm64-fips
+        env:
+          FIPS: "true"
+          CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
+          CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud"
+          TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud:git-${BUILDKITE_COMMIT:0:12}"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-arm64-fips'
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix.groups}} {{matrix.sudo}}
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        retry:
+          automatic:
+            limit: 1
+        agents:
+          provider: "aws"
+          image: "${IMAGE_UBUNTU_ARM64_FIPS}"
+          instanceType: "m6g.2xlarge"
+        matrix:
+          setup:
+            sudo:
+              - "false"
+              - "true"
+            groups:
+              - fleet
+
+  - label: ESS FIPS stack cleanup
+    depends_on:
+      - integration-tests-ubuntu-fips
+    allow_dependency_failure: true
+    command: |
+      buildkite-agent artifact download "test_infra/ess/**" . --step "integration-fips-ess"
+      ls -lah test_infra/ess
+      .buildkite/scripts/steps/ess_down.sh
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true
+
+  - label: Aggregate test reports
+    depends_on:
+      - integration-tests-ubuntu-fips
+    allow_dependency_failure: true
+    command: |
+      buildkite-agent artifact download "build/*.xml" .
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true
+    soft_fail:
+      - exit_status: "*"
+    plugins:
+      - elastic/vault-secrets#v0.1.0:
+          path: "kv/ci-shared/platform-ingest/buildkite_analytics_token"
+          field: "token"
+          env_var: "BUILDKITE_ANALYTICS_TOKEN"
+      - test-collector#v1.11.0:
+          files: "build/*.xml"
+          format: "junit"
+          branches: "main"
+          debug: true
diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -72,3 +72,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" ]]; then
     release_manager_login
   fi
 fi
+
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == "integration-fips-cloud-image" ]]; then
+    docker_login
+fi
diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml
@@ -177,3 +177,8 @@ steps:
     depends_on:
       - int-packaging
     command: "buildkite-agent pipeline upload .buildkite/bk.integration.pipeline.yml"
+
+  - label: "Triggering custom FIPS integration tests"
+    depends_on:
+      - int-packaging
+    command: "buildkite-agent pipeline upload .buildkite/bk.integration-fips.pipeline.yml"
diff --git a/.buildkite/scripts/buildkite-integration-tests.sh b/.buildkite/scripts/buildkite-integration-tests.sh
@@ -15,6 +15,11 @@ if [ -z "$TEST_SUDO" ]; then
   exit 1
 fi
 
+if [ "${FIPS:-false}" == "true" ]; then
+  echo "~~~FIPS: Checking msft-go is installed"
+  GOEXPERIMENT=systemcrypto go version
+fi
+
 if [ "$TEST_SUDO" == "true" ]; then
   echo "Re-initializing ASDF. The user is changed to root..."
   export ASDF_DATA_DIR="/opt/buildkite-agent/.asdf"
@@ -50,7 +55,7 @@ GOTEST_ARGS=(-tags integration -test.shuffle on -test.timeout 2h0m0s)
 if [ -n "$TEST_NAME_PATTERN" ]; then
   GOTEST_ARGS+=(-run="${TEST_NAME_PATTERN}")
 fi
-GOTEST_ARGS+=("github.com/elastic/elastic-agent/testing/integration" -v -args "-integration.groups=${GROUP_NAME}" "-integration.sudo=${TEST_SUDO}")
+GOTEST_ARGS+=("github.com/elastic/elastic-agent/testing/integration" -v -args "-integration.groups=${GROUP_NAME}" "-integration.sudo=${TEST_SUDO}" "-integration.fips=${FIPS:-false}")
 
 set +e
 TEST_BINARY_NAME="elastic-agent" AGENT_VERSION="${AGENT_VERSION}" SNAPSHOT=true \
diff --git a/.ci/updatecli/updatecli-bump-vm-images.yml b/.ci/updatecli/updatecli-bump-vm-images.yml
@@ -63,3 +63,13 @@ targets:
       file: .buildkite/bk.integration.pipeline.yml
       matchpattern: '(IMAGE_.+): "platform-ingest-elastic-agent-(.+)-(.+)"'
       replacepattern: '$1: "platform-ingest-elastic-agent-$2-{{ source "latestVersion" }}"'
+
+  update-buildkite-bk.integration-fips.pipeline:
+    name: "Update .buildkite/bk.integration-fips.pipeline.yml"
+    sourceid: latestVersion
+    scmid: githubConfig
+    kind: file
+    spec:
+      file: .buildkite/bk.integration-fips.pipeline.yml
+      matchpattern: '(IMAGE_.+): "platform-ingest-elastic-agent-(.+)-(.+)"'
+      replacepattern: '$1: "platform-ingest-elastic-agent-$2-{{ source "latestVersion" }}"'
diff --git a/.tool-versions b/.tool-versions
@@ -1,3 +1,2 @@
 mage 1.14.0
-golang 1.24.0
 terraform 1.9.3
diff --git a/magefile.go b/magefile.go
@@ -979,6 +979,37 @@ func (Cloud) Image(ctx context.Context) {
 	Package(ctx)
 }
 
+// Load loads an artifact as a docker image.
+// Looks in build/distributions for an elastic-agent-cloud*.docker.tar.gz artifact and imports it as docker.elastic.co/beats-ci/elastic-agent-cloud:$VERSION
+// DOCKER_IMPORT_SOURCE - override source for import
+func (Cloud) Load() error {
+	snapshot := os.Getenv(snapshotEnv)
+	defer os.Setenv(snapshotEnv, snapshot)
+	os.Setenv(snapshotEnv, "true")
+
+	version := getVersion()
+
+	// Need to get the FIPS env var flag to see if we are using the normal source cloud image name, or the FIPS variant
+	fips := os.Getenv(fipsEnv)
+	defer os.Setenv(fipsEnv, fips)
+	fipsVal, err := strconv.ParseBool(fips)
+	if err != nil {
+		fipsVal = false
+	}
+	os.Setenv(fipsEnv, strconv.FormatBool(fipsVal))
+	devtools.FIPSBuild = fipsVal
+
+	source := "build/distributions/elastic-agent-cloud-" + version + "-linux-" + runtime.GOARCH + ".docker.tar.gz"
+	if fipsVal {
+		source = "build/distributions/elastic-agent-fips-cloud-" + version + "-linux-" + runtime.GOARCH + ".docker.tar.gz"
+	}
+	if envSource, ok := os.LookupEnv("DOCKER_IMPORT_SOURCE"); ok && envSource != "" {
+		source = envSource
+	}
+
+	return sh.RunV("docker", "image", "load", "-i", source)
+}
+
 // Push builds a cloud image tags it correctly and pushes to remote image repo.
 // Previous login to elastic registry is required!
 func (Cloud) Push() error {
@@ -998,7 +1029,20 @@ func (Cloud) Push() error {
 		tag = fmt.Sprintf("%s-%s-%d", version, commit, time)
 	}
 
+	// Need to get the FIPS env var flag to see if we are using the normal source cloud image name, or the FIPS variant
+	fips := os.Getenv(fipsEnv)
+	defer os.Setenv(fipsEnv, fips)
+	fipsVal, err := strconv.ParseBool(fips)
+	if err != nil {
+		fipsVal = false
+	}
+	os.Setenv(fipsEnv, strconv.FormatBool(fipsVal))
+	devtools.FIPSBuild = fipsVal
+
 	sourceCloudImageName := fmt.Sprintf("docker.elastic.co/beats-ci/elastic-agent-cloud:%s", version)
+	if fipsVal {
+		sourceCloudImageName = fmt.Sprintf("docker.elastic.co/beats-ci/elastic-agent-fips-cloud:%s", version)
+	}
 	var targetCloudImageName string
 	if customImage, isPresent := os.LookupEnv("CI_ELASTIC_AGENT_DOCKER_IMAGE"); isPresent && len(customImage) > 0 {
 		targetCloudImageName = fmt.Sprintf("%s:%s", customImage, tag)
@@ -1007,7 +1051,7 @@ func (Cloud) Push() error {
 	}
 
 	fmt.Printf(">> Setting a docker image tag to %s\n", targetCloudImageName)
-	err := sh.RunV("docker", "tag", sourceCloudImageName, targetCloudImageName)
+	err = sh.RunV("docker", "tag", sourceCloudImageName, targetCloudImageName)
 	if err != nil {
 		return fmt.Errorf("Failed setting a docker image tag: %w", err)
 	}
diff --git a/testing/integration/fleetserver_fips_test.go b/testing/integration/fleetserver_fips_test.go
@@ -0,0 +1,88 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License 2.0;
+// you may not use this file except in compliance with the Elastic License 2.0.
+
+//go:build integration
+
+package integration
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/elastic/elastic-agent-libs/kibana"
+	"github.com/elastic/elastic-agent/pkg/testing/define"
+	"github.com/elastic/elastic-agent/pkg/testing/tools/fleettools"
+)
+
+const cloudAgentPolicyID = "policy-elastic-agent-on-cloud"
+
+// TestFIPSAgentConnectingToFIPSFleetServerInECHFRH ensures that a FIPS-capable Elastic Agent
+// running in an ECH FRH (FedRamp High) environment is able to successfully connect to its
+// own local Fleet Server instance (which, by definition should also be FIPS-capable and
+// running in the ECH FRH environment).
+func TestFIPSAgentConnectingToFIPSFleetServerInECHFRH(t *testing.T) {
+	info := define.Require(t, define.Requirements{
+		Group: Fleet,
+		Stack: &define.Stack{},
+		OS: []define.OS{
+			{Type: define.Linux},
+		},
+		Sudo:  false,
+		Local: true,
+
+		// Ensures the test will run in a FIPS-configured environment against a
+		// deployment in ECH that's running a FIPS-capable integrations server.
+		FIPS: true,
+	})
+
+	fleetServerHost, err := fleettools.DefaultURL(t.Context(), info.KibanaClient)
+	require.NoError(t, err)
+	statusUrl, err := url.JoinPath(fleetServerHost, "/api/status")
+	require.NoError(t, err)
+
+	resp, err := http.Get(statusUrl)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	var body struct {
+		Name   string `json:"name"`
+		Status string `json:"status"`
+	}
+	decoder := json.NewDecoder(resp.Body)
+	err = decoder.Decode(&body)
+	require.NoError(t, err)
+
+	require.Equalf(t, "HEALTHY", body.Status, "response status code: %d", resp.StatusCode)
+
+	// Get all Agents
+	ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+	defer cancel()
+	agents, err := info.KibanaClient.ListAgents(ctx, kibana.ListAgentsRequest{})
+	require.NoError(t, err)
+
+	// Find Fleet Server's own Agent and get its status and whether it's
+	// FIPS-capable
+	//var agentStatus string
+	var agentIsFIPS bool
+	for _, item := range agents.Items {
+		if item.PolicyID == cloudAgentPolicyID {
+			t.Logf("Found fleet-server entry: %+v", item)
+			//agentStatus = item.Status
+			agentIsFIPS = item.LocalMetadata.Elastic.Agent.FIPS
+			break
+		}
+	}
+
+	// Check that this Agent is online (i.e. healthy) and is FIPS-capable. This
+	// will prove that a FIPS-capable Agent is able to connect to a FIPS-capable
+	// Fleet Server, with both running in ECH.
+	require.Equal(t, true, agentIsFIPS)
+	//require.Equal(t, "online", agentStatus) // FIXME: Uncomment after https://github.com/elastic/apm-server/issues/17063 is resolved
+}

Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,2 @@`
`1`	`1`	`mage 1.14.0`
`2`		`-golang 1.24.0`
`3`	`2`	`terraform 1.9.3`