[CI][FIPS] Reconfigure pipeline to use Staging GovCloud/FRH ESS environment (elastic#9198)

* Get Staging FRH/GovCloud API key from Vault

* Point Elastic Cloud API endpoint for TF provider to Staging FRH/GovCloud environment

* Updating Vault path for API key

* [Test commit] No-op change to trigger FIPS tests

* Set ESS region to Staging GovCloud region

* Check if variable is set before using it

* Use default value

* Parameterize deployment template ID

* Debugging terraform

* Forgot to pass deployment template ID

* Use TF_VAR_* instead of adding new env vars

* Fix deployment template ID

* Set ES and Kibana Docker image URLs

* Removing unrelated changes

* Set -fips suffix on default docker images

* Remove commented out lines

* Bring back ESS_REGION env var

* Define default ESS_REGION so it's not unbound

* Restoring file from main

* Try to set STACK_BUILD_ID to ""

* [Testing] Hardcoding stack build ID

* Only use STACK_VERSION when FIPS=true

* Revert "[Testing] Hardcoding stack build ID"

This reverts commit fb275fb.

* Remove unnecessary line

Author: ycombinator

.buildkite/bk.integration-fips.pipeline.yml

@@ -11,9 +11,9 @@ env:
 # This section is used to define the plugins that will be used in the pipeline.
 # See https://buildkite.com/docs/pipelines/integrations/plugins/using#using-yaml-anchors-with-plugins
 common:
-  - vault_ec_key_prod: &vault_ec_key_prod
+  - vault_ec_key_staging_frh_gov: &vault_ec_key_staging_frh_gov
       elastic/vault-secrets#v0.1.0:
-        path: "kv/ci-shared/platform-ingest/platform-ingest-ec-prod"
+        path: "kv/ci-shared/platform-ingest/platform-ingest-ec-staging-gov"
         field: "apiKey"
         env_var: "EC_API_KEY"
 
@@ -23,7 +23,11 @@ steps:
     env:
       ASDF_TERRAFORM_VERSION: 1.9.2
       FIPS: "true"
+      EC_ENDPOINT: "https://api.staging.elastic-gov.com"
+      ESS_REGION: "us-gov-east-1"
+      TF_VAR_deployment_template_id: "aws-general-purpose"
       TF_VAR_integration_server_docker_image: "docker.elastic.co/beats-ci/elastic-agent-cloud-fips:git-${BUILDKITE_COMMIT:0:12}"
+      TF_VAR_docker_images_name_suffix: "-fips"
     command: |
       source .buildkite/scripts/steps/ess_start.sh
     artifact_paths:
@@ -33,7 +37,7 @@ steps:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
       useCustomGlobalHooks: true
     plugins:
-      - *vault_ec_key_prod
+      - *vault_ec_key_staging_frh_gov
 
   - group: "fips:Stateful:Ubuntu"
     key: integration-tests-ubuntu-fips
@@ -61,7 +65,7 @@ steps:
           image: "${IMAGE_UBUNTU_X86_64_FIPS}"
           instanceType: "m5.2xlarge"
         plugins:
-          - *vault_ec_key_prod
+          - *vault_ec_key_staging_frh_gov
         matrix:
           setup:
             sudo:
@@ -91,7 +95,7 @@ steps:
           image: "${IMAGE_UBUNTU_ARM64_FIPS}"
           instanceType: "m6g.2xlarge"
         plugins:
-          - *vault_ec_key_prod
+          - *vault_ec_key_staging_frh_gov
         matrix:
           setup:
             sudo:
@@ -118,7 +122,7 @@ steps:
           image: "${IMAGE_UBUNTU_X86_64_FIPS}"
           instanceType: "m5.2xlarge"
         plugins:
-          - *vault_ec_key_prod
+          - *vault_ec_key_staging_frh_gov
 
   - label: ESS FIPS stack cleanup
     depends_on:
@@ -132,7 +136,7 @@ steps:
       image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.5"
       useCustomGlobalHooks: true
     plugins:
-      - *vault_ec_key_prod
+      - *vault_ec_key_staging_frh_gov
 
   - label: Aggregate test reports
     depends_on:

.buildkite/scripts/steps/ess_start.sh

@@ -6,8 +6,14 @@ source .buildkite/scripts/steps/fleet.sh
 
 STACK_VERSION="$(jq -r '.version' .package-version)"
 STACK_BUILD_ID="$(jq -r '.stack_build_id' .package-version)"
+if [[ "${FIPS:-false}" == "true" ]]; then
+  # FRH testing environment does not have same stack build IDs as CFT environment so
+  # we just go with the STACK_VERSION.
+  STACK_BUILD_ID=""
+fi
+ESS_REGION="${ESS_REGION:-gcp-us-west2}"
 
-ess_up "$STACK_VERSION" "$STACK_BUILD_ID"
+ess_up "$STACK_VERSION" "$STACK_BUILD_ID" "$ESS_REGION"
 
 preinstall_fleet_packages
 

.buildkite/scripts/steps/integration_tests_tf.sh

@@ -25,6 +25,11 @@ fi
 # This file is managed by an automation (mage integration:UpdateAgentPackageVersion) that check if the snapshot is ready.
 STACK_VERSION="$(jq -r '.version' .package-version)"
 STACK_BUILD_ID="$(jq -r '.stack_build_id' .package-version)"
+if [[ "${FIPS:-false}" == "true" ]]; then
+  # FRH testing environment does not have same stack build IDs as CFT environment so
+  # we just go with the STACK_VERSION.
+  STACK_BUILD_ID=""
+fi
 
 echo "~~~ Building test binaries"
 mage build:testBinaries

test_infra/ess/deployment.tf

@@ -58,6 +58,12 @@ variable "kibana_docker_image" {
   description = "Docker image override for kibana"
 }
 
+variable "docker_images_name_suffix" {
+  type        = string
+  default     = ""
+  description = "Suffix to append to the docker images names"
+}
+
 resource "random_uuid" "deployment_suffix" {
 }
 
@@ -80,9 +86,9 @@ locals {
     yamldecode(file("${path.module}/../../pkg/testing/ess/create_deployment_csp_configuration.yaml")))
 
   images_version = coalesce(var.stack_build_id, var.stack_version)
-  integration_server_docker_image = coalesce(var.integration_server_docker_image, local.ess_properties.docker.integration_server_image, "docker.elastic.co/cloud-release/elastic-agent-cloud:${local.images_version}")
-  elasticsearch_docker_image = coalesce(var.elasticsearch_docker_image, local.ess_properties.docker.elasticsearch_image, "docker.elastic.co/cloud-release/elasticsearch-cloud-ess:${local.images_version}")
-  kibana_docker_image = coalesce(var.kibana_docker_image, local.ess_properties.docker.kibana_image, "docker.elastic.co/cloud-release/kibana-cloud:${local.images_version}")
+  integration_server_docker_image = coalesce(var.integration_server_docker_image, local.ess_properties.docker.integration_server_image, "docker.elastic.co/cloud-release/elastic-agent-cloud${var.docker_images_name_suffix}:${local.images_version}")
+  elasticsearch_docker_image = coalesce(var.elasticsearch_docker_image, local.ess_properties.docker.elasticsearch_image, "docker.elastic.co/cloud-release/elasticsearch-cloud-ess${var.docker_images_name_suffix}:${local.images_version}")
+  kibana_docker_image = coalesce(var.kibana_docker_image, local.ess_properties.docker.kibana_image, "docker.elastic.co/cloud-release/kibana-cloud${var.docker_images_name_suffix}:${local.images_version}")
 }
 
 # If we have defined a stack version, validate that this version exists on that region and return it.