diff --git a/.buildkite/bk.fips-integration.pipeline.yml b/.buildkite/bk.fips-integration.pipeline.yml new file mode 100644 index 00000000000..c60865c60a6 --- /dev/null +++ b/.buildkite/bk.fips-integration.pipeline.yml @@ -0,0 +1,178 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json + +env: + DOCKER_REGISTRY: "docker.elastic.co" + VAULT_PATH: "kv/ci-shared/observability-ingest/cloud/gcp" + ASDF_MAGE_VERSION: 1.14.0 + FIPS: "true" + CUSTOM_IMAGE_TAG: "git-${BUILDKITE_COMMIT:0:12}" + CI_ELASTIC_AGENT_DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud" + + IMAGE_UBUNTU_FIPS: "platform-ingest-fleet-server-ubuntu-2204-fips" # image may only be in aws? + IMAGE_UBUNTU_2404_X86_64: "platform-ingest-elastic-agent-ubuntu-2404-1744855248" + +steps: + - label: Build and push custom elastic-agent image + key: integration-fips-cloud-image + env: + ASDF_TERRAFORM_VERSION: 1.9.2 + command: | + #!/usr/bin/env bash + set -euo pipefail + mage cloud:image + mage cloud:push + agents: + provider: "gcp" + machineType: "n1-standard-8" + image: "${IMAGE_UBUNTU_2404_X86_64}" + + - label: Start ESS stack for integration tests + key: integration-fips-ess + depends_on: + - integration-fips-cloud-image + env: + ASDF_TERRAFORM_VERSION: 1.9.2 + TF_VAR_integration_server_docker_image: "${CI_ELASTIC_AGENT_DOCKER_IMAGE}:${CUSTOM_IMAGE_TAG}" + command: | + #!/usr/bin/env bash + set -euo pipefail + source .buildkite/scripts/steps/ess_start.sh + artifact_paths: + - test_infra/ess/*.tfstate + - test_infra/ess/*.lock.hcl + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5" + useCustomGlobalHooks: true + + - group: "fips:Stateful:Ubuntu" + key: integration-tests-ubuntu-fips + depends_on: + - integration-fips-ess + steps: + - label: "fips:non-sudo:{{matrix}}" + depends_on: + - packaging-ubuntu-x86-64-fips + command: | + buildkite-agent artifact download build/distributions/** . --step 'packaging-ubuntu-x86-64-fips' + .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} false + artifact_paths: + - build/** + - build/diagnostics/** + retry: + automatic: + limit: 1 + agents: + provider: "gcp" + machineType: "n1-standard-8" + image: "${IMAGE_UBUNTU_2404_X86_64}" + matrix: + - default + + - label: "fips:sudo:{{matrix}}" + depends_on: + - packaging-ubuntu-x86-64-fips + command: | + buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64-fips + .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true + artifact_paths: + - build/** + - build/diagnostics/** + retry: + automatic: + limit: 1 + agents: + provider: "gcp" + machineType: "n1-standard-8" + image: "${IMAGE_UBUNTU_2404_X86_64}" + matrix: + - default + #- upgrade + #- upgrade-flavor + #- standalone-upgrade + #- fleet + #- fleet-endpoint-security + #- fleet-airgapped + #- fleet-airgapped-privileged + #- fleet-privileged + #- fleet-upgrade-to-pr-build + #- install-uninstall + #- fqdn + #- deb + #- container + + #- group: "Kubernetes" + # key: integration-tests-kubernetes + # depends_on: + # - integration-ess + # - packaging-containers-x86-64 + # steps: + # - label: "{{matrix.version}}:amd64:{{matrix.variants}}" + # env: + # K8S_VERSION: "{{matrix.version}}" + # ASDF_KIND_VERSION: "0.27.0" + # DOCKER_VARIANTS: "{{matrix.variants}}" + # TARGET_ARCH: "amd64" + # AGENT_VERSION: "9.0.0-SNAPSHOT" # Remove agent pinning once 9.0.0 is released + # command: | + # buildkite-agent artifact download build/distributions/*-linux-amd64.docker.tar.gz . --step 'packaging-containers-x86-64' + # .buildkite/scripts/steps/integration_tests_tf.sh kubernetes false + # artifact_paths: + # - build/** + # - build/diagnostics/** + # - build/*.pod_logs_dump/* + # retry: + # automatic: + # limit: 1 + # agents: + # provider: "gcp" + # machineType: "n1-standard-4" + # image: "${IMAGE_UBUNTU_2404_X86_64}" + # diskSizeGb: 80 + # matrix: + # setup: + # variants: + # - "basic,slim,complete,service,elastic-otel-collector" + # - "wolfi,slim-wolfi,complete-wolfi,elastic-otel-collector-wolfi" + # version: + # - v1.27.16 + # - v1.28.9 + # - v1.29.8 + # - v1.30.8 + # - v1.31.0 + # - v1.32.0 + + - label: ESS stack cleanup + depends_on: + - integration-tests-ubuntu + - integration-tests-win + - integration-tests-rhel8 + - integration-tests-kubernetes + allow_dependency_failure: true + command: | + buildkite-agent artifact download "test_infra/ess/**" . --step "integration-ess" + ls -lah test_infra/ess + .buildkite/scripts/steps/ess_down.sh + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5" + useCustomGlobalHooks: true + + - label: Aggregate test reports + # Warning: The key has a hook in pre-command + key: aggregate-reports-fips + depends_on: + - integration-tests-ubuntu-fips + #- integration-tests-kubernetes + allow_dependency_failure: true + command: | + buildkite-agent artifact download "build/*.xml" . + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5" + useCustomGlobalHooks: true + soft_fail: + - exit_status: "*" + plugins: + - test-collector#v1.10.1: + files: "build/*.xml" + format: "junit" + branches: "main" + debug: true diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 83622d4e81b..c1ce11cdcfb 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -72,7 +72,7 @@ if [[ "$BUILDKITE_STEP_KEY" == *"aggregate-reports"* ]]; then export BUILDKITE_ANALYTICS_TOKEN fi -if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent-binary-dra" || ("$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == "integration-fips-cloud-image") ]]; then if command -v docker &>/dev/null; then docker_login else diff --git a/.buildkite/integration.pipeline.yml b/.buildkite/integration.pipeline.yml index e4f09c24735..12dfb7ea14e 100644 --- a/.buildkite/integration.pipeline.yml +++ b/.buildkite/integration.pipeline.yml @@ -201,3 +201,8 @@ steps: depends_on: - int-packaging command: "buildkite-agent pipeline upload .buildkite/bk.integration.pipeline.yml" + + - label: "Triggering custom ECH integration tests" + depends_on: + - int-packaging + command: "buildkite-agent pipeline upload .buildkite/bk.fips-integration.pipeline.yml" diff --git a/.buildkite/scripts/buildkite-integration-tests.sh b/.buildkite/scripts/buildkite-integration-tests.sh index 3642ee06b36..ce79359dc6b 100755 --- a/.buildkite/scripts/buildkite-integration-tests.sh +++ b/.buildkite/scripts/buildkite-integration-tests.sh @@ -22,6 +22,12 @@ if [ "$TEST_SUDO" == "true" ]; then source .buildkite/hooks/pre-command || echo "No pre-command hook found" fi +INTEGRATION_TEST_ARGS="-integration.groups=\"${GROUP_NAME}\" -integration.sudo=\"${TEST_SUDO}\"" +if [[ "${FIPS:-false}" == "true" ]]; then + echo "FIPS testing detected" + #INTEGRATION_TEST_ARGS+=" -integration.fips=true" # FIXME re-enable once adding this filter picks up tests +fi + # Make sure that all tools are installed asdf install @@ -51,7 +57,16 @@ outputJSON="build/${fully_qualified_group_name}.integration.out.json" echo "~~~ Integration tests: ${GROUP_NAME}" set +e -TEST_BINARY_NAME="elastic-agent" AGENT_VERSION="${AGENT_VERSION}" SNAPSHOT=true gotestsum --no-color -f standard-quiet --junitfile "${outputXML}" --jsonfile "${outputJSON}" -- -tags integration -test.shuffle on -test.timeout 2h0m0s github.com/elastic/elastic-agent/testing/integration -v -args -integration.groups="${GROUP_NAME}" -integration.sudo="${TEST_SUDO}" +TEST_BINARY_NAME="elastic-agent" AGENT_VERSION="${AGENT_VERSION}" SNAPSHOT=true \ + gotestsum --no-color -f standard-quiet \ + --junitfile "${outputXML}" \ + --jsonfile "${outputJSON}" \ + -- \ + -tags integration -test.shuffle on -test.timeout 2h0m0s \ + github.com/elastic/elastic-agent/testing/integration \ + -v \ + -args "${INTEGRATION_TEST_ARGS}" + TESTS_EXIT_STATUS=$? set -e diff --git a/.buildkite/scripts/steps/integration_tests_tf.sh b/.buildkite/scripts/steps/integration_tests_tf.sh index ab51a46e2a1..c8227ba4c2d 100755 --- a/.buildkite/scripts/steps/integration_tests_tf.sh +++ b/.buildkite/scripts/steps/integration_tests_tf.sh @@ -21,6 +21,10 @@ if [ -z "$TEST_SUDO" ]; then exit 1 fi +if [[ ${FIPS:-false} == "true " ]]; then + echo "FIPS Integration tests detected." +fi + # Override the agent package version using a string with format .. # There is a time when the snapshot is not built yet, so we cannot use the latest version automatically # This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready. diff --git a/magefile.go b/magefile.go index a0422d71ba6..a2b85cc2af3 100644 --- a/magefile.go +++ b/magefile.go @@ -1001,7 +1001,19 @@ func (Cloud) Push() error { tag = fmt.Sprintf("%s-%s-%d", version, commit, time) } + fips := os.Getenv(fipsEnv) + defer os.Setenv(fipsEnv, fips) + fipsVal, err := strconv.ParseBool(fips) + if err != nil { + fipsVal = false + } + os.Setenv(fipsEnv, strconv.FormatBool(fipsVal)) + devtools.FIPSBuild = fipsVal + sourceCloudImageName := fmt.Sprintf("docker.elastic.co/beats-ci/elastic-agent-cloud:%s", version) + if fipsVal { + sourceCloudImageName = fmt.Sprintf("docker.elastic.co/beats-ci/elastic-agent-fips-cloud:%s", version) + } var targetCloudImageName string if customImage, isPresent := os.LookupEnv("CI_ELASTIC_AGENT_DOCKER_IMAGE"); isPresent && len(customImage) > 0 { targetCloudImageName = fmt.Sprintf("%s:%s", customImage, tag) @@ -1010,7 +1022,7 @@ func (Cloud) Push() error { } fmt.Printf(">> Setting a docker image tag to %s\n", targetCloudImageName) - err := sh.RunV("docker", "tag", sourceCloudImageName, targetCloudImageName) + err = sh.RunV("docker", "tag", sourceCloudImageName, targetCloudImageName) if err != nil { return fmt.Errorf("Failed setting a docker image tag: %w", err) }