Add basic fips cloudE2E test (#4983)

Verify that FIPS capable artifacts work in cloud deployments.
Correct openapi status endpoint return values.

Author: michel-laterman

.buildkite/hooks/pre-command

@@ -46,17 +46,19 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-package-mbp" ]]; then
 fi
 
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" || "$BUILDKITE_PIPELINE_SLUG" == "fleet-server-perf-tests" ]]; then
-  if [[ "$BUILDKITE_STEP_KEY" == "publish" || "$BUILDKITE_STEP_KEY" == "cloud-e2e-test" || "$BUILDKITE_STEP_KEY" == "create-image" ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == "publish" || "$BUILDKITE_STEP_KEY" == "cloud-e2e-test" || "$BUILDKITE_STEP_KEY" == "cloud-e2e-fips-test"  || "$BUILDKITE_STEP_KEY" == "create-image" ]]; then
     export DOCKER_USERNAME_SECRET=$(retry 5 vault kv get -field user "${DOCKER_REGISTRY_SECRET_PATH}")
     export DOCKER_PASSWORD_SECRET=$(retry 5 vault kv get -field password "${DOCKER_REGISTRY_SECRET_PATH}")
     docker login -u "${DOCKER_USERNAME_SECRET}" -p "${DOCKER_PASSWORD_SECRET}" "${DOCKER_REGISTRY}" 2>/dev/null
   fi
 fi
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" && "$BUILDKITE_STEP_KEY" == "cloud-e2e-test" ]]; then
-  export EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey "${EC_KEY_SECRET_PATH}")
-  # Environment variables required by the Elastic Cloud service deployer
-  export EC_API_KEY=${EC_API_KEY_SECRET}
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "fleet-server" ]]; then
+  if [[ "$BUILDKITE_STEP_KEY" == "cloud-e2e-test" || "$BUILDKITE_STEP_KEY" == "cloud-e2e-fips-test"  ]]; then
+    export EC_API_KEY_SECRET=$(retry 5 vault kv get -field apiKey "${EC_KEY_SECRET_PATH}")
+    # Environment variables required by the Elastic Cloud service deployer
+    export EC_API_KEY=${EC_API_KEY_SECRET}
+  fi
 fi
 
 # BK analytics

.buildkite/pipeline.yml

@@ -44,6 +44,7 @@ steps:
 
   - group: "Performance test"
     key: "performance-test"
+    depends_on: "check"
     steps:
       - label: "Run go benchmark for PR branch"
         key: "go-benchmark-pr"
@@ -82,10 +83,9 @@ steps:
         agents:
           provider: "gcp"
 
-    depends_on: "check"
-
   - group: "Run tests"
     key: "tests"
+    depends_on: "check"
     steps:
       - label: ":smartbear-testexecute: Run unit tests"
         key: unit-test
@@ -209,7 +209,28 @@ steps:
             allow_failure: false
           - step: "e2e-test"
             allow_failure: false
-    depends_on: "check"
+
+      - label: ":gcloud: Cloud e2e FIPS Test"
+        key: "cloud-e2e-fips-test"
+        env:
+          DOCKER_REGISTRY: "docker.elastic.co"
+          DOCKER_BASE_IMAGE: "docker.elastic.co/cloud-release/elastic-agent-fips-cloud"
+          DOCKER_IMAGE: "docker.elastic.co/beats-ci/elastic-agent-fips-cloud"
+          DOCKER_IMAGE_TAG: "pr-${BUILDKITE_PULL_REQUEST}-${BUILDKITE_COMMIT:0:12}"
+          SNAPSHOT: "true"
+          PLATFORMS: "linux/amd64"
+          TF_VAR_pull_request: "${BUILDKITE_PULL_REQUEST}"
+          FIPS: "true"
+        command: ".buildkite/scripts/cloud_e2e_test.sh"
+        agents:
+          provider: "gcp"
+        depends_on:
+          - step: "unit-test"
+            allow_failure: false
+          - step: "int-test"
+            allow_failure: false
+          - step: "e2e-test"
+            allow_failure: false
 
   - label: ":docker: Publish docker image"
     key: "publish"

internal/pkg/api/openapi.gen.go

@@ -1233,7 +1233,7 @@ func Test_SmokeTest_CheckinPollTimeout(t *testing.T) {
 			AgentId:  agentID,
 			Message:  "test-message",
 			Type:     api.ACTIONRESULT,
-			Subtype:  api.ACKNOWLEDGED,
+			Subtype:  api.EventSubtypeACKNOWLEDGED,
 		}
 		ev := api.AckRequest_Events_Item{}
 		err := ev.FromGenericEvent(event)
@@ -1365,7 +1365,7 @@ func Test_SmokeTest_CheckinPollShutdown(t *testing.T) {
 			AgentId:  agentID,
 			Message:  "test-message",
 			Type:     api.ACTIONRESULT,
-			Subtype:  api.ACKNOWLEDGED,
+			Subtype:  api.EventSubtypeACKNOWLEDGED,
 		}
 		ev := api.AckRequest_Events_Item{}
 		err := ev.FromGenericEvent(event)

internal/pkg/server/fleet_integration_test.go

@@ -82,8 +82,10 @@ const (
 	envDockerImage = "DOCKER_IMAGE"
 	// envDockerTag is used to indicate tag for images produced by the docker:image target. Defaults to version. It
 	envDockerTag = "DOCKER_IMAGE_TAG"
-	// envDockerBaseImage is the image:tag for the base elastic-agent-cloud images used by e2e tests.
+	// envDockerBaseImage is the base image for elastic-agent-cloud images used by e2e tests.
 	envDockerBaseImage = "DOCKER_BASE_IMAGE"
+	// envDockerBaseImageTag is the tag for the base image used by e2e tests.
+	envDockerBaseImageTag = "DOCKER_BASE_IMAGE_TAG"
 )
 
 // const and vars used by magefile.
@@ -997,12 +999,17 @@ func (Docker) Image() error {
 
 // Push pushs an image created by docker:image to the registry.
 // FIPS may be used to push a FIPS capable image.
+// DOCKER_IMAGE may be used to specify the image name.
 // DOCKER_IMAGE_TAG may be used to specify the image tag.
 func (Docker) Push() error {
 	image := dockerImage
 	if isFIPS() {
 		image += "-fips"
 	}
+	if v, ok := os.LookupEnv(envDockerImage); ok && v != "" {
+		image = v
+	}
+
 	version := getVersion()
 	if v, ok := os.LookupEnv(envDockerTag); ok && v != "" {
 		version = v
@@ -1015,6 +1022,7 @@ func (Docker) Push() error {
 // This step requires a coverage enabled binary to be used.
 // FIPS is used to control if a FIPS compliant image should be created.
 // DOCKER_BASE_IMAGE may be used to specify the elastic-agent base image. docker.elastic.co/cloud-release/elastic-agent-cloud by default.
+// DOCKER_BASE_IMAGE_TAG may be used to specify the elastic-agent base image tag. Uses the ELASTICESRCH version from dev-tools/integration/.env.
 // DOCKER_IMAGE is used to specify the resulting image name.
 // DOCKER_IMAGE_TAG is used to specify the resulting image tag.
 func (Docker) CustomAgentImage() error {
@@ -1023,16 +1031,21 @@ func (Docker) CustomAgentImage() error {
 		return fmt.Errorf("unable to read env file: %w", err)
 	}
 
-	baseImage := "docker.elastic.co/cloud-release/elastic-agent-cloud:" + env["ELASTICSEARCH_VERSION"]
+	baseImage := "docker.elastic.co/cloud-release/elastic-agent-cloud"
 	if v, ok := os.LookupEnv(envDockerBaseImage); ok && v != "" {
 		baseImage = v
 	}
+	baseImageTag := env["ELASTICSEARCH_VERSION"]
+	if v, ok := os.LookupEnv(envDockerBaseImageTag); ok && v != "" {
+		baseImageTag = v
+	}
+
 	dockerEnv := map[string]string{"DOCKER_BUILDKIT": "1"}
-	err = sh.RunWithV(dockerEnv, "docker", "pull", "--platform", "linux/"+runtime.GOARCH, baseImage)
+	err = sh.RunWithV(dockerEnv, "docker", "pull", "--platform", "linux/"+runtime.GOARCH, baseImage+":"+baseImageTag)
 	if err != nil {
 		return fmt.Errorf("failed to pull image: %w", err)
 	}
-	vcsRef, err := sh.OutputWith(dockerEnv, "docker", "inspect", "-f", "{{index .Config.Labels \"org.label-schema.vcs-ref\" }}", baseImage)
+	vcsRef, err := sh.OutputWith(dockerEnv, "docker", "inspect", "-f", "{{index .Config.Labels \"org.label-schema.vcs-ref\" }}", baseImage+":"+baseImageTag)
 	if err != nil {
 		return fmt.Errorf("unable to find vcs-ref label: %w", err)
 	}
@@ -1050,7 +1063,7 @@ func (Docker) CustomAgentImage() error {
 	}
 	err = sh.RunWithV(dockerEnv, "docker", "build",
 		"-f", filepath.Join("dev-tools", "e2e", "Dockerfile"),
-		"--build-arg", "ELASTIC_AGENT_IMAGE="+baseImage,
+		"--build-arg", "ELASTIC_AGENT_IMAGE="+baseImage+":"+baseImageTag,
 		"--build-arg", "STACK_VERSION="+getVersion(),
 		"--build-arg", "VCS_REF_SHORT="+vcsRef[:6],
 		"--build-arg", "FLEET_FIPS="+fips,
@@ -2027,6 +2040,7 @@ func (Test) CloudE2E() {
 // DOCKER_IMAGE can be used to specify the custom integration server image.
 // DOCKER_IMAGE_TAG can be used to specify the tag of the custom integration server.
 func (Test) CloudE2EUp() error {
+	os.Setenv(envSnapshot, "true")
 	imageName := dockerImage
 	imageTag := getVersion()
 
@@ -2081,16 +2095,35 @@ func (Test) CloudE2EDown() error {
 
 // CloudE2ERun runs tests against the remote cloud deployment.
 func (Test) CloudE2ERun() error {
-	url, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "fleet_url")
+	fleetURL, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "fleet_url")
 	if err != nil {
 		return fmt.Errorf("unable to retrive fleet-server cloud url: %w", err)
 	}
 
+	kibanaURL, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "kibana_url")
+	if err != nil {
+		return fmt.Errorf("unable to retrive kibana cloud url: %w", err)
+	}
+
+	user, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "elasticsearch_username")
+	if err != nil {
+		return fmt.Errorf("unable to retrive es username: %w", err)
+	}
+	pass, err := sh.Output("terraform", "output", "--raw", "--state="+filepath.Join("dev-tools", "cloud", "terraform", "terraform.tfstate"), "elasticsearch_password")
+	if err != nil {
+		return fmt.Errorf("unable to retrive es password: %w", err)
+	}
+
 	var b bytes.Buffer
 	w := io.MultiWriter(&b, os.Stdout)
 	cmd := exec.Command("go", "test", "-v", "-timeout", "30m", "-tags=cloude2e", "-count=1", "-p", "1", "./...")
 	cmd.Dir = "testing"
-	cmd.Env = append(os.Environ(), "FLEET_SERVER_URL="+url)
+	cmd.Env = append(os.Environ(),
+		"FLEET_SERVER_URL="+fleetURL,
+		"KIBANA_URL="+kibanaURL,
+		"ELASTIC_USER="+user,
+		"ELASTIC_PASS="+pass,
+	)
 	cmd.Stdout = w
 	cmd.Stderr = w
 	err = cmd.Run()

magefile.go

@@ -86,14 +86,14 @@ components:
             A Unit state that fleet-server may report.
             Unit state is defined in the elastic-agent-client specification.
           enum:
-            - starting
-            - configuring
-            - healthy
-            - degraded
-            - failed
-            - stopping
-            - stopped
-            - unknown
+            - STARTING
+            - CONFIGURING
+            - HEALTHY
+            - DEGRADED
+            - FAILED
+            - STOPPING
+            - STOPPED
+            - UNKNOWN
         version:
           $ref: "#/components/schemas/statusResponseVersion"
     enrollMetadata: