[DO NOT MERGE] Trying out some fixes for PR: #8035

.buildkite/bk.fips-integration.pipeline.yml

@@ -0,0 +1,178 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+env:
+  DOCKER_REGISTRY: "docker.elastic.co"
+  VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp"
+  ASDF_MAGE_VERSION: 1.14.0
+  FIPS: "true"
+  CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}"
+  CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud"
+
+  IMAGE_UBUNTU_FIPS: "platform-ingest-fleet-server-ubuntu-2204-fips" # image may only be in aws?
+  IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1744855248"
+
+steps:
+  - label: Build and push custom elastic-agent image
+    key: integration-fips-cloud-image
+    env:
+      ASDF_TERRAFORM_VERSION: 1.9.2
+    command: |
+      #!/usr/bin/env bash
+      set -euo pipefail
+      mage cloud:image
+      mage cloud:push
+    agents:
+      provider: "gcp"
+      machineType: "n1-standard-8"
+      image: "${IMAGE_UBUNTU_2404_X86_64}"
+
+  - label: Start ESS stack for integration tests
+    key: integration-fips-ess
+    depends_on:
+      - integration-fips-cloud-image
+    env:
+      ASDF_TERRAFORM_VERSION: 1.9.2
+      TF_VAR_integration_server_docker_image: "${CI_ELASTIC_AGENT_DOCKER_IMAGE}:${CUSTOM_IMAGE_TAG}"
+    command: |
+      #!/usr/bin/env bash
+      set -euo pipefail
+      source .buildkite/scripts/steps/ess_start.sh
+    artifact_paths:
+      - test_infra/ess/*.tfstate
+      - test_infra/ess/*.lock.hcl
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true
+
+  - group: "fips:Stateful:Ubuntu"
+    key: integration-tests-ubuntu-fips
+    depends_on:
+      - integration-fips-ess
+    steps:
+      - label: "fips:non-sudo:{{matrix}}"
+        depends_on:
+          - packaging-ubuntu-x86-64-fips
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64-fips'
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        retry:
+          automatic:
+            limit: 1
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          image: "${IMAGE_UBUNTU_2404_X86_64}"
+        matrix:
+          - default
+
+      - label: "fips:sudo:{{matrix}}"
+        depends_on:
+          - packaging-ubuntu-x86-64-fips
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64-fips
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        retry:
+          automatic:
+            limit: 1
+        agents:
+          provider: "gcp"
+          machineType: "n1-standard-8"
+          image: "${IMAGE_UBUNTU_2404_X86_64}"
+        matrix:
+          - default
+          #- upgrade
+          #- upgrade-flavor
+          #- standalone-upgrade
+          #- fleet
+          #- fleet-endpoint-security
+          #- fleet-airgapped
+          #- fleet-airgapped-privileged
+          #- fleet-privileged
+          #- fleet-upgrade-to-pr-build
+          #- install-uninstall
+          #- fqdn
+          #- deb
+          #- container
+
+  #- group: "Kubernetes"
+  #  key: integration-tests-kubernetes
+  #  depends_on:
+  #    - integration-ess
+  #    - packaging-containers-x86-64
+  #  steps:
+  #    - label: "{{matrix.version}}:amd64:{{matrix.variants}}"
+  #      env:
+  #        K8S_VERSION: "{{matrix.version}}"
+  #        ASDF_KIND_VERSION: "0.27.0"
+  #        DOCKER_VARIANTS: "{{matrix.variants}}"
+  #        TARGET_ARCH: "amd64"
+  #        AGENT_VERSION: "9.0.0-SNAPSHOT" # Remove agent pinning once 9.0.0 is released
+  #      command: |
+  #        buildkite-agent artifact download build/distributions/*-linux-amd64.docker.tar.gz . --step 'packaging-containers-x86-64'
+  #        .buildkite/scripts/steps/integration_tests_tf.sh kubernetes false
+  #      artifact_paths:
+  #        - build/**
+  #        - build/diagnostics/**
+  #        - build/*.pod_logs_dump/*
+  #      retry:
+  #        automatic:
+  #          limit: 1
+  #      agents:
+  #        provider: "gcp"
+  #        machineType: "n1-standard-4"
+  #        image: "${IMAGE_UBUNTU_2404_X86_64}"
+  #        diskSizeGb: 80
+  #      matrix:
+  #        setup:
+  #          variants:
+  #          - "basic,slim,complete,service,elastic-otel-collector"
+  #          - "wolfi,slim-wolfi,complete-wolfi,elastic-otel-collector-wolfi"
+  #          version:
+  #          - v1.27.16
+  #          - v1.28.9
+  #          - v1.29.8
+  #          - v1.30.8
+  #          - v1.31.0
+  #          - v1.32.0
+
+  - label: ESS stack cleanup
+    depends_on:
+      - integration-tests-ubuntu
+      - integration-tests-win
+      - integration-tests-rhel8
+      - integration-tests-kubernetes
+    allow_dependency_failure: true
+    command: |
+      buildkite-agent artifact download "test_infra/ess/**" . --step "integration-ess"
+      ls -lah test_infra/ess
+      .buildkite/scripts/steps/ess_down.sh
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true
+
+  - label: Aggregate test reports
+    # Warning: The key has a hook in pre-command
+    key: aggregate-reports-fips
+    depends_on:
+      - integration-tests-ubuntu-fips
+      #- integration-tests-kubernetes
+    allow_dependency_failure: true
+    command: |
+      buildkite-agent artifact download "build/*.xml" .
+    agents:
+      image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
+      useCustomGlobalHooks: true
+    soft_fail:
+      - exit_status: "*"
+    plugins:
+      - test-collector#v1.10.1:
+          files: "build/*.xml"
+          format: "junit"
+          branches: "main"
+          debug: true

.buildkite/hooks/pre-command

@@ -72,7 +72,7 @@ if [[ "$BUILDKITE_STEP_KEY" == *"aggregate-reports"* ]]; then
   export BUILDKITE_ANALYTICS_TOKEN
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" || ("$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == "integration-fips-cloud-image") ]]; then
   if command -v docker &>/dev/null; then
     docker_login
   else

.buildkite/integration.pipeline.yml

@@ -201,3 +201,8 @@ steps:
     depends_on:
       - int-packaging
     command: "buildkite-agent pipeline upload .buildkite/bk.integration.pipeline.yml"
+
+  - label: "Triggering custom ECH integration tests"
+    depends_on:
+      - int-packaging
+    command: "buildkite-agent pipeline upload .buildkite/bk.fips-integration.pipeline.yml"

.buildkite/scripts/buildkite-integration-tests.sh

@@ -22,6 +22,12 @@ if [ "$TEST_SUDO" == "true" ]; then
   source .buildkite/hooks/pre-command || echo "No pre-command hook found"
 fi
 
+INTEGRATION_TEST_ARGS="-integration.groups=\"${GROUP_NAME}\" -integration.sudo=\"${TEST_SUDO}\""
+if [[ "${FIPS:-false}" == "true" ]]; then
+    echo "FIPS testing detected"
+    #INTEGRATION_TEST_ARGS+=" -integration.fips=true" # FIXME re-enable once adding this filter picks up tests
+fi
+
 # Make sure that all tools are installed
 asdf install
 
@@ -51,7 +57,16 @@ outputJSON="build/${fully_qualified_group_name}.integration.out.json"
 echo "~~~ Integration tests: ${GROUP_NAME}"
 
 set +e
-TEST_BINARY_NAME="elastic-agent" AGENT_VERSION="${AGENT_VERSION}" SNAPSHOT=true gotestsum --no-color -f standard-quiet --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- -tags integration -test.shuffle on -test.timeout 2h0m0s github.com/elastic/elastic-agent/testing/integration -v -args -integration.groups="${GROUP_NAME}" -integration.sudo="${TEST_SUDO}"
+TEST_BINARY_NAME="elastic-agent" AGENT_VERSION="${AGENT_VERSION}" SNAPSHOT=true \
+  gotestsum --no-color -f standard-quiet \
+  --junitfile "${outputXML}" \
+  --jsonfile "${outputJSON}" \
+  -- \
+    -tags integration -test.shuffle on -test.timeout 2h0m0s \
+    github.com/elastic/elastic-agent/testing/integration \
+    -v \
+    -args "${INTEGRATION_TEST_ARGS}"
+
 TESTS_EXIT_STATUS=$?
 set -e
 

.buildkite/scripts/steps/integration_tests_tf.sh

@@ -21,6 +21,10 @@ if [ -z "$TEST_SUDO" ]; then
   exit 1
 fi
 
+if [[ ${FIPS:-false} == "true " ]]; then
+    echo "FIPS Integration tests detected."
+fi
+
 # Override the agent package version using a string with format <major>.<minor>.<patch>
 # There is a time when the snapshot is not built yet, so we cannot use the latest version automatically
 # This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready.

magefile.go

@@ -1001,7 +1001,19 @@ func (Cloud) Push() error {
 		tag = fmt.Sprintf("%s-%s-%d", version, commit, time)
 	}
 
+	fips := os.Getenv(fipsEnv)
+	defer os.Setenv(fipsEnv, fips)
+	fipsVal, err := strconv.ParseBool(fips)
+	if err != nil {
+		fipsVal = false
+	}
+	os.Setenv(fipsEnv, strconv.FormatBool(fipsVal))
+	devtools.FIPSBuild = fipsVal
+
 	sourceCloudImageName := fmt.Sprintf("docker.elastic.co/beats-ci/elastic-agent-cloud:%s", version)
+	if fipsVal {
+		sourceCloudImageName = fmt.Sprintf("docker.elastic.co/beats-ci/elastic-agent-fips-cloud:%s", version)
+	}
 	var targetCloudImageName string
 	if customImage, isPresent := os.LookupEnv("CI_ELASTIC_AGENT_DOCKER_IMAGE"); isPresent && len(customImage) > 0 {
 		targetCloudImageName = fmt.Sprintf("%s:%s", customImage, tag)
@@ -1010,7 +1022,7 @@ func (Cloud) Push() error {
 	}
 
 	fmt.Printf(">> Setting a docker image tag to %s\n", targetCloudImageName)
-	err := sh.RunV("docker", "tag", sourceCloudImageName, targetCloudImageName)
+	err = sh.RunV("docker", "tag", sourceCloudImageName, targetCloudImageName)
 	if err != nil {
 		return fmt.Errorf("Failed setting a docker image tag: %w", err)
 	}