Integration Test: upgrade Integrations Server in ECH (#8417)

* WIP: Implementing UpgradeDeployment method

* Adding AvailableVersions method to StatefulProvisioner

* WIP: adding test for upgrading integration server in ECH

* Ensure test runs against FRH ECH region

* WIP

* WIP

* Add test for generating upgrade deployment request

* Flesh out test some more

* Fixing filename in reference

* Make DeploymentStatus contain status and health

* Cleanup deployment after test

* Add error handling

* Make linter happy

* Fix typo

* Fix check

* Adding logging

* Filter by min/max versions

* Fix call

* Running non-sudo upgrade group tests

* Running upgrade FIPS tests

* Remove non-sudo upgrade group from regular integration tests

* Fix env var for ECH region and default it to gcp-us-west2

* Use env var for ECH API key

* Run ECH deployment upgrade test if PR label is present

* Fix missing key check

* Add label on PR that bumps integration test versions

* Export EC_API_KEY

* WIP: debugging

* Append query string to URL after joining paths

* Fix up logic in doGet to construct URL without escaping ?

* WIP: Remove some debug logging

* Add bounds check

* Fix start index math

* Removing debug logs

* WIP: don't clean up deployment so we can investigate

* Add doPut method on client

* Remove debugging statements

* Fix build tag

* Change group for testing in PR

* Cleanup

* Add comment

* Cleanup

* Update deployment version after upgrade

* Test group in PR

* Fix unit test

* Undo unintentional changes

Author: ycombinator

.buildkite/bk.integration-fips.pipeline.yml

@@ -107,6 +107,25 @@ steps:
             groups:
               - fleet
 
+      - label: "fips:upgrade-ech-deployment"
+        if: build.env("BUILDKITE_PULL_REQUEST") != "false" &&  build.env("GITHUB_PR_LABELS") =~ /.*(Testing:run:TestUpgradeIntegrationsServer).*/
+        env:
+          FIPS: "true"
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
+        command: |
+          export EC_API_KEY=$(retry -t 5 -- vault kv get -field=apiKey kv/ci-shared/platform-ingest/platform-ingest-ec-prod)
+          .buildkite/scripts/buildkite-integration-tests.sh ech-deployment false
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        retry:
+          automatic:
+            limit: 1
+        agents:
+          provider: "aws"
+          image: "${IMAGE_UBUNTU_X86_64_FIPS}"
+          instanceType: "m5.2xlarge"
+
   - label: ESS FIPS stack cleanup
     depends_on:
       - integration-tests-ubuntu-fips

.ci/updatecli/updatecli-bump-stack-version.yml

@@ -26,12 +26,12 @@ sources:
       key: .build
 targets:
   updateStackImages:
-    name: "Update create_deployment_csp_configuration.yaml"
+    name: "Update deployment_csp_configuration.yaml"
     sourceid: lastStableStackSnapshot
     scmid: githubConfig
     kind: file
     spec:
-      file: "pkg/testing/ess/create_deployment_csp_configuration.yaml"
+      file: "pkg/testing/ess/deployment_csp_configuration.yaml"
       matchpattern: '(\S*_image: ".*docker.*):(.+)"'
       replacepattern: '$1:{{ source "lastStableStackSnapshot" }}-SNAPSHOT"'
 actions:
@@ -48,4 +48,4 @@ actions:
         - "Team:Elastic-Agent-Control-Plane"
       title: '[{{ requiredEnv "BRANCH_NAME" }}][Automation] Bump stack images versions to {{ source "lastStableStackSnapshot" }}'
       description: |
-        Updates `create_deployment_csp_configuration.yml`. It is used to pin stable images versions for integration tests.
+        Updates `deployment_csp_configuration.yml`. It is used to pin stable images versions for integration tests.

.github/workflows/bump-agent-versions.sh

@@ -38,6 +38,7 @@ else
        --label 'update-versions' \
        --label 'skip-changelog' \
        --label 'backport-skip' \
+       --label 'Testing:run:TestUpgradeIntegrationsServer' \
        --repo $GITHUB_REPOSITORY)
     echo "pr=$pr" >> "$GITHUB_OUTPUT" # set the step output for Slack notifications
     echo "Created a PR with the an update: $pr"

.mergify.yml

@@ -19,24 +19,24 @@ defaults:
       labels:
         - "backport"
 pull_request_rules:
-  - name: automatic approval for updatecli pull requests with changes in .buildkite or create_deployment_csp_configuration.yaml
+  - name: automatic approval for updatecli pull requests with changes in .buildkite or deployment_csp_configuration.yaml
     conditions:
       - author=github-actions[bot]
       - check-success=buildkite/elastic-agent
-      - or: 
-        - files~=^pkg/testing/ess/create_deployment_csp_configuration.yaml$
+      - or:
+        - files~=^pkg/testing/ess/deployment_csp_configuration.yaml$
         - files~=^.buildkite/(pipeline.yml|bk.integration.pipeline.yml)$
       - head~=^updatecli_.*
     actions:
       review:
         type: APPROVE
         message: Automatically approving mergify
-  - name: automatic squash and merge with success checks and the files matching the regex .buildkite or create_deployment_csp_configuration.yaml is modified.
+  - name: automatic squash and merge with success checks and the files matching the regex .buildkite or deployment_csp_configuration.yaml is modified.
     conditions:
       - author=github-actions[bot]
       - check-success=buildkite/elastic-agent
-      - or: 
-        - files~=^pkg/testing/ess/create_deployment_csp_configuration.yaml$
+      - or:
+        - files~=^pkg/testing/ess/deployment_csp_configuration.yaml$
         - files~=^.buildkite/(pipeline.yml|bk.integration.pipeline.yml)$
       - head~=^updatecli_.*
       - "#approved-reviews-by>=1"
@@ -46,8 +46,8 @@ pull_request_rules:
   - name: rebase pull requests for the VM autobump and ESS version pinning
     conditions:
       - author=github-actions[bot]
-      - or: 
-        - files~=^pkg/testing/ess/create_deployment_csp_configuration.yaml$
+      - or:
+        - files~=^pkg/testing/ess/deployment_csp_configuration.yaml$
         - files~=^.buildkite/(pipeline.yml|bk.integration.pipeline.yml)$
       - head~=^updatecli_.*
       - "#check-failure>0"

pkg/testing/common/stack.go

@@ -4,7 +4,9 @@
 
 package common
 
-import "context"
+import (
+	"context"
+)
 
 // Stack is a created stack.
 type Stack struct {
@@ -72,9 +74,12 @@ type StackProvisioner interface {
 	// Create creates a stack.
 	Create(ctx context.Context, request StackRequest) (Stack, error)
 
-	// WaitForReady should block until the stack is ready or the context is cancelled.
+	// WaitForReady should block until the stack is ready and healthy or the context is cancelled.
 	WaitForReady(ctx context.Context, stack Stack) (Stack, error)
 
 	// Delete deletes the stack.
 	Delete(ctx context.Context, stack Stack) error
+
+	// Upgrade upgrades the stack to a new version.
+	Upgrade(ctx context.Context, stack Stack, newVersion string) error
 }

pkg/testing/ess/client.go

@@ -10,11 +10,14 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+
+	"github.com/elastic/elastic-agent/pkg/testing/common"
 )
 
 type Client struct {
 	config *Config
 	client *http.Client
+	logger common.Logger
 }
 
 func NewClient(config Config) *Client {
@@ -28,11 +31,14 @@ func NewClient(config Config) *Client {
 	return c
 }
 
+func (c *Client) SetLogger(logger common.Logger) {
+	c.logger = logger
+}
+
 func (c *Client) doGet(ctx context.Context, relativeUrl string) (*http.Response, error) {
-	u, err := url.JoinPath(c.config.BaseUrl, relativeUrl)
-	if err != nil {
-		return nil, fmt.Errorf("unable to create API URL: %w", err)
-	}
+	// We don't use url.JoinPath because it escapes the "?" in
+	// relativeUrl if it contains a query string
+	u := c.config.BaseUrl + "/" + relativeUrl
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
 	if err != nil {
@@ -45,12 +51,20 @@ func (c *Client) doGet(ctx context.Context, relativeUrl string) (*http.Response,
 }
 
 func (c *Client) doPost(ctx context.Context, relativeUrl, contentType string, body io.Reader) (*http.Response, error) {
+	return c.doWrite(ctx, http.MethodPost, relativeUrl, contentType, body)
+}
+
+func (c *Client) doPut(ctx context.Context, relativeUrl, contentType string, body io.Reader) (*http.Response, error) {
+	return c.doWrite(ctx, http.MethodPut, relativeUrl, contentType, body)
+}
+
+func (c *Client) doWrite(ctx context.Context, method, relativeUrl, contentType string, body io.Reader) (*http.Response, error) {
 	u, err := url.JoinPath(c.config.BaseUrl, relativeUrl)
 	if err != nil {
 		return nil, fmt.Errorf("unable to create API URL: %w", err)
 	}
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, body)
+	req, err := http.NewRequestWithContext(ctx, method, u, body)
 	if err != nil {
 		return nil, fmt.Errorf("unable to create POST request: %w", err)
 	}