From f7ea7a590fef3d64b8116408715a3e951803eff7 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Apr 2025 09:58:39 -0700 Subject: [PATCH 01/14] WIP: scaffolding for test --- testing/integration/fleetserver_fips_test.go | 33 ++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 testing/integration/fleetserver_fips_test.go diff --git a/testing/integration/fleetserver_fips_test.go b/testing/integration/fleetserver_fips_test.go new file mode 100644 index 00000000000..43b5dbc86ac --- /dev/null +++ b/testing/integration/fleetserver_fips_test.go @@ -0,0 +1,33 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +//go:build integration + +package integration + +// This test has nothing to do with the locally-built Agent artifact! It merely +// orchestrates resources in ECH (Elastic Cloud - Hosted). The purpose of this test +// is to ensure that a FIPS-capable Elastic Agent running in the ECH FRH (FedRamp High) +// environment is able to successfully connect to it's own local Fleet Server instance +// (which, by definition should also be FIPS-capable and running in the ECH FRH environment). + +func TestFIPSAgentConnectingToFIPSFleetServerInECH(t *testing.T) { + info := define.Require(t, define.Requirements{ + Group: Fleet, + Stack: &define.Stack{}, + Sudo: false, + Local: true, + }) + + // Re-use ECH API key used by integration tests to spin up a deployment + // in ECH. The deployment must contain an Integrations Server, which includes + // an Agent running with a local Fleet Server. Note that we want to use a + // FIPS-capable build of Elastic Agent (with Fleet Server) for this deployment. + // Further, the Fleet Server must be configured with FIPS-compliant TLS (TLSv1.2 + // and TLSv1.3 and appropriate ciphers). + + // Once the deployment is completely spun up, ensure that the Agent in the + // deployment is healthy and connected to Fleet. This will prove that a FIPS-capable + // Agent is able to connect to a FIPS-capable Fleet Server, with both running in ECH. +} From b384ac286c6ad62e3d02956b8efc02b068ec9a03 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Apr 2025 10:06:10 -0700 Subject: [PATCH 02/14] WIP: Print out env vars --- testing/integration/fleetserver_fips_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/testing/integration/fleetserver_fips_test.go b/testing/integration/fleetserver_fips_test.go index 43b5dbc86ac..2962fe1affa 100644 --- a/testing/integration/fleetserver_fips_test.go +++ b/testing/integration/fleetserver_fips_test.go @@ -6,6 +6,8 @@ package integration +import "os" + // This test has nothing to do with the locally-built Agent artifact! It merely // orchestrates resources in ECH (Elastic Cloud - Hosted). The purpose of this test // is to ensure that a FIPS-capable Elastic Agent running in the ECH FRH (FedRamp High) @@ -21,7 +23,12 @@ func TestFIPSAgentConnectingToFIPSFleetServerInECH(t *testing.T) { }) // Re-use ECH API key used by integration tests to spin up a deployment - // in ECH. The deployment must contain an Integrations Server, which includes + // in ECH. + for _, envVar := range os.Environ() { + t.Log(envVar) + } + + //The deployment must contain an Integrations Server, which includes // an Agent running with a local Fleet Server. Note that we want to use a // FIPS-capable build of Elastic Agent (with Fleet Server) for this deployment. // Further, the Fleet Server must be configured with FIPS-compliant TLS (TLSv1.2 From c8c5ad944729e3be7aa6ab2bfa130b9716086f9b Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Apr 2025 10:12:26 -0700 Subject: [PATCH 03/14] Update comment --- testing/integration/fleetserver_fips_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testing/integration/fleetserver_fips_test.go b/testing/integration/fleetserver_fips_test.go index 2962fe1affa..fcf6e868c52 100644 --- a/testing/integration/fleetserver_fips_test.go +++ b/testing/integration/fleetserver_fips_test.go @@ -23,7 +23,8 @@ func TestFIPSAgentConnectingToFIPSFleetServerInECH(t *testing.T) { }) // Re-use ECH API key used by integration tests to spin up a deployment - // in ECH. + // in ECH. Deployment must use an image of Integration Server that has + // FIPS-capable artifacts. for _, envVar := range os.Environ() { t.Log(envVar) } From 4e97536f74fadf3a60797919c823d54aa5c8afc3 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Apr 2025 11:01:35 -0700 Subject: [PATCH 04/14] Running mage fmt --- testing/integration/fleetserver_fips_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/testing/integration/fleetserver_fips_test.go b/testing/integration/fleetserver_fips_test.go index fcf6e868c52..083eb354fca 100644 --- a/testing/integration/fleetserver_fips_test.go +++ b/testing/integration/fleetserver_fips_test.go @@ -6,7 +6,12 @@ package integration -import "os" +import ( + "os" + "testing" + + "github.com/elastic/elastic-agent/pkg/testing/define" +) // This test has nothing to do with the locally-built Agent artifact! It merely // orchestrates resources in ECH (Elastic Cloud - Hosted). The purpose of this test From dec47ab7f36d07cb1c52dbbd6f28f3ed7038f336 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Apr 2025 11:16:31 -0700 Subject: [PATCH 05/14] Fix syntax errors --- testing/integration/fleetserver_fips_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/integration/fleetserver_fips_test.go b/testing/integration/fleetserver_fips_test.go index 083eb354fca..94c7f2fb405 100644 --- a/testing/integration/fleetserver_fips_test.go +++ b/testing/integration/fleetserver_fips_test.go @@ -20,7 +20,7 @@ import ( // (which, by definition should also be FIPS-capable and running in the ECH FRH environment). func TestFIPSAgentConnectingToFIPSFleetServerInECH(t *testing.T) { - info := define.Require(t, define.Requirements{ + _ = define.Require(t, define.Requirements{ Group: Fleet, Stack: &define.Stack{}, Sudo: false, From 934db825435a188892b513f48a3e97c6df53cb77 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 30 Apr 2025 13:08:27 -0700 Subject: [PATCH 06/14] Update comments --- testing/integration/fleetserver_fips_test.go | 44 ++++++++++---------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/testing/integration/fleetserver_fips_test.go b/testing/integration/fleetserver_fips_test.go index 94c7f2fb405..4ed9f4277b1 100644 --- a/testing/integration/fleetserver_fips_test.go +++ b/testing/integration/fleetserver_fips_test.go @@ -2,42 +2,44 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -//go:build integration +//go:build integration && frh package integration import ( - "os" "testing" "github.com/elastic/elastic-agent/pkg/testing/define" ) -// This test has nothing to do with the locally-built Agent artifact! It merely -// orchestrates resources in ECH (Elastic Cloud - Hosted). The purpose of this test -// is to ensure that a FIPS-capable Elastic Agent running in the ECH FRH (FedRamp High) -// environment is able to successfully connect to it's own local Fleet Server instance -// (which, by definition should also be FIPS-capable and running in the ECH FRH environment). - -func TestFIPSAgentConnectingToFIPSFleetServerInECH(t *testing.T) { +// IMPORTANT: This file has build tags of integration && frh. This means the tests in +// this file will NOT be run along with other integration tests (which use deployments +// in the ESS production CFT region). The tests in this file need to be run using +// deployments in an ECH FRH (FedRamp High) region. This region can be specified using +// the following environment variables: +// TEST_INTEG_AUTH_ESS_FRH_URL (default: https://api.staging.elastic-gov.com/) +// TEST_INTEG_AUTH_ESS_FRH_REGION (default: us-gov-east-1) +// TEST_INTEG_AUTH_ESS_FRH_APIKEY + +// TestFIPSAgentConnectingToFIPSFleetServerInECHFRH ensures that a FIPS-capable Elastic Agent +// running in an ECH FRH (FedRamp High) environment is able to successfully connect to its +// own local Fleet Server instance (which, by definition should also be FIPS-capable and +// running in the ECH FRH environment). +// NOTE: This test has nothing to do with the locally-built Agent artifact! It merely +// orchestrates resources in ECH (Elastic Cloud - Hosted). + +func TestFIPSAgentConnectingToFIPSFleetServerInECHFRH(t *testing.T) { _ = define.Require(t, define.Requirements{ Group: Fleet, - Stack: &define.Stack{}, + Stack: nil, Sudo: false, Local: true, }) - // Re-use ECH API key used by integration tests to spin up a deployment - // in ECH. Deployment must use an image of Integration Server that has - // FIPS-capable artifacts. - for _, envVar := range os.Environ() { - t.Log(envVar) - } - - //The deployment must contain an Integrations Server, which includes - // an Agent running with a local Fleet Server. Note that we want to use a - // FIPS-capable build of Elastic Agent (with Fleet Server) for this deployment. - // Further, the Fleet Server must be configured with FIPS-compliant TLS (TLSv1.2 + // Connect to ECH FRH environment and spin up deployment. The deployment must + // contain an Integrations Server, which includes an Agent running with a local + // Fleet Server. Note that we will need to use a FIPS-capable build of Elastic Agent + // (with Fleet Server) for this deployment. Further, the Fleet Server must be configured with FIPS-compliant TLS (TLSv1.2 // and TLSv1.3 and appropriate ciphers). // Once the deployment is completely spun up, ensure that the Agent in the From 99827e5f2b89acbfacd8affd92ccb8644afbb76f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 9 May 2025 13:26:32 -0700 Subject: [PATCH 07/14] Flesh out test --- testing/integration/fleetserver_fips_test.go | 83 +++++++++++++++----- 1 file changed, 62 insertions(+), 21 deletions(-) diff --git a/testing/integration/fleetserver_fips_test.go b/testing/integration/fleetserver_fips_test.go index 4ed9f4277b1..5df132424e0 100644 --- a/testing/integration/fleetserver_fips_test.go +++ b/testing/integration/fleetserver_fips_test.go @@ -2,47 +2,88 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -//go:build integration && frh +//go:build integration package integration import ( + "context" + "encoding/json" + "net/http" + "net/url" + "os" "testing" + "time" + + "github.com/stretchr/testify/require" "github.com/elastic/elastic-agent/pkg/testing/define" ) -// IMPORTANT: This file has build tags of integration && frh. This means the tests in -// this file will NOT be run along with other integration tests (which use deployments -// in the ESS production CFT region). The tests in this file need to be run using -// deployments in an ECH FRH (FedRamp High) region. This region can be specified using -// the following environment variables: -// TEST_INTEG_AUTH_ESS_FRH_URL (default: https://api.staging.elastic-gov.com/) -// TEST_INTEG_AUTH_ESS_FRH_REGION (default: us-gov-east-1) -// TEST_INTEG_AUTH_ESS_FRH_APIKEY +const cloudAgentPolicyID = "policy-elastic-agent-on-cloud" // TestFIPSAgentConnectingToFIPSFleetServerInECHFRH ensures that a FIPS-capable Elastic Agent // running in an ECH FRH (FedRamp High) environment is able to successfully connect to its // own local Fleet Server instance (which, by definition should also be FIPS-capable and // running in the ECH FRH environment). -// NOTE: This test has nothing to do with the locally-built Agent artifact! It merely -// orchestrates resources in ECH (Elastic Cloud - Hosted). - func TestFIPSAgentConnectingToFIPSFleetServerInECHFRH(t *testing.T) { - _ = define.Require(t, define.Requirements{ + info := define.Require(t, define.Requirements{ Group: Fleet, - Stack: nil, + Stack: &define.Stack{}, + OS: []define.OS{ + {Type: define.Linux}, + }, Sudo: false, Local: true, + + // Ensures the test will run in a FIPS-configured environment against a + // deployment in ECH that's running a FIPS-capable integrations server. + FIPS: true, }) - // Connect to ECH FRH environment and spin up deployment. The deployment must - // contain an Integrations Server, which includes an Agent running with a local - // Fleet Server. Note that we will need to use a FIPS-capable build of Elastic Agent - // (with Fleet Server) for this deployment. Further, the Fleet Server must be configured with FIPS-compliant TLS (TLSv1.2 + // Further, the Fleet Server must be configured with FIPS-compliant TLS (TLSv1.2 // and TLSv1.3 and appropriate ciphers). - // Once the deployment is completely spun up, ensure that the Agent in the - // deployment is healthy and connected to Fleet. This will prove that a FIPS-capable - // Agent is able to connect to a FIPS-capable Fleet Server, with both running in ECH. + // Check that the Fleet Server in the deployment is healthy + fleetServerHost := os.Getenv("FLEETSERVER_HOST") + statusUrl, err := url.JoinPath(fleetServerHost, "/api/status") + require.NoError(t, err) + + resp, err := http.Get(statusUrl) + require.NoError(t, err) + defer resp.Body.Close() + + var body struct { + Name string `json:"name"` + Status string `json:"status"` + } + decoder := json.NewDecoder(resp.Body) + err = decoder.Decode(&body) + require.NoError(t, err) + + require.Equal(t, "HEALTHY", body.Status) + + // Get all Agents + ctx, cancel := context.WithTimeout(5*time.Second, context.Background()) + defer cancel() + agents, err := info.KibanaClient.ListAgents(ctx) + require.NoError(t, err) + + // Find Fleet Server's own Agent and get its status and whether it's + // FIPS-capable + var agentStatus string + var agentIsFIPS bool + for _, item := range agents.Items { + if item.PolicyID == cloudAgentPolicyID { + agentStatus = item.Status + // TODO: requires change in elastic-agent-libs + //agentIsFIPS = item.LocalMetadata.Elastic.Agent.FIPS + } + } + + // Check that this Agent is online (i.e. healthy) and is FIPS-capable. This + // will prove that a FIPS-capable Agent is able to connect to a FIPS-capable + // Fleet Server, with both running in ECH. + require.Equal(t, "online", agentStatus) + require.Equal(t, true, agentIsFIPS) } From c441a7b30a07eb9f40c35b5f5775cc6dc21a2a2a Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 9 May 2025 13:37:07 -0700 Subject: [PATCH 08/14] Reference PR from elastic-agent-libs --- testing/integration/fleetserver_fips_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testing/integration/fleetserver_fips_test.go b/testing/integration/fleetserver_fips_test.go index 5df132424e0..6424978e21c 100644 --- a/testing/integration/fleetserver_fips_test.go +++ b/testing/integration/fleetserver_fips_test.go @@ -76,7 +76,8 @@ func TestFIPSAgentConnectingToFIPSFleetServerInECHFRH(t *testing.T) { for _, item := range agents.Items { if item.PolicyID == cloudAgentPolicyID { agentStatus = item.Status - // TODO: requires change in elastic-agent-libs + // FIXME: needs https://github.com/elastic/elastic-agent-libs/pull/320 + // to be merged and a new elastic-agent-libs to be released. //agentIsFIPS = item.LocalMetadata.Elastic.Agent.FIPS } } From d368605fa689fec9a116c92340bf3114c0f468af Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 9 May 2025 13:46:15 -0700 Subject: [PATCH 09/14] Removing comment --- testing/integration/fleetserver_fips_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/testing/integration/fleetserver_fips_test.go b/testing/integration/fleetserver_fips_test.go index 6424978e21c..95804042f7b 100644 --- a/testing/integration/fleetserver_fips_test.go +++ b/testing/integration/fleetserver_fips_test.go @@ -76,8 +76,6 @@ func TestFIPSAgentConnectingToFIPSFleetServerInECHFRH(t *testing.T) { for _, item := range agents.Items { if item.PolicyID == cloudAgentPolicyID { agentStatus = item.Status - // FIXME: needs https://github.com/elastic/elastic-agent-libs/pull/320 - // to be merged and a new elastic-agent-libs to be released. //agentIsFIPS = item.LocalMetadata.Elastic.Agent.FIPS } } From 6f711f46622d18a0a93ba4f060991c36b59b9b95 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 9 May 2025 16:20:04 -0700 Subject: [PATCH 10/14] Remove comment that is no longer necessary --- testing/integration/fleetserver_fips_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/testing/integration/fleetserver_fips_test.go b/testing/integration/fleetserver_fips_test.go index 95804042f7b..9157ba29a3f 100644 --- a/testing/integration/fleetserver_fips_test.go +++ b/testing/integration/fleetserver_fips_test.go @@ -41,9 +41,6 @@ func TestFIPSAgentConnectingToFIPSFleetServerInECHFRH(t *testing.T) { FIPS: true, }) - // Further, the Fleet Server must be configured with FIPS-compliant TLS (TLSv1.2 - // and TLSv1.3 and appropriate ciphers). - // Check that the Fleet Server in the deployment is healthy fleetServerHost := os.Getenv("FLEETSERVER_HOST") statusUrl, err := url.JoinPath(fleetServerHost, "/api/status") From 3cd18e5fce1d74b4b27a06a93c0fe2ccf495a31d Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 9 May 2025 16:35:56 -0700 Subject: [PATCH 11/14] Export INTEGRATIONS_SERVER_HOST from ESS deployment's terraform output --- .buildkite/scripts/steps/ess.ps1 | 1 + .buildkite/scripts/steps/ess.sh | 1 + .buildkite/scripts/steps/ess_start.sh | 1 + test_infra/ess/output.tf | 4 ++++ testing/integration/fleetserver_fips_test.go | 2 +- 5 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.buildkite/scripts/steps/ess.ps1 b/.buildkite/scripts/steps/ess.ps1 index d8371f16f19..0a11a49fb9e 100644 --- a/.buildkite/scripts/steps/ess.ps1 +++ b/.buildkite/scripts/steps/ess.ps1 @@ -42,6 +42,7 @@ function ess_up { $Env:KIBANA_HOST = & terraform output -raw kibana_endpoint $Env:KIBANA_USERNAME = $Env:ELASTICSEARCH_USERNAME $Env:KIBANA_PASSWORD = $Env:ELASTICSEARCH_PASSWORD + $Env:INTEGRATIONS_SERVER_HOST = & terraform output -raw integrations_server_endpoint Pop-Location } diff --git a/.buildkite/scripts/steps/ess.sh b/.buildkite/scripts/steps/ess.sh index 563648f85a7..d1792a8bb29 100755 --- a/.buildkite/scripts/steps/ess.sh +++ b/.buildkite/scripts/steps/ess.sh @@ -40,6 +40,7 @@ function ess_up() { export KIBANA_HOST=$(terraform output -raw kibana_endpoint) export KIBANA_USERNAME=$ELASTICSEARCH_USERNAME export KIBANA_PASSWORD=$ELASTICSEARCH_PASSWORD + export INTEGRATIONS_SERVER_HOST=$(terraform output -raw integrations_server_endpoint) popd } diff --git a/.buildkite/scripts/steps/ess_start.sh b/.buildkite/scripts/steps/ess_start.sh index 79322ef665c..0e9d67c96d9 100755 --- a/.buildkite/scripts/steps/ess_start.sh +++ b/.buildkite/scripts/steps/ess_start.sh @@ -19,3 +19,4 @@ buildkite-agent meta-data set "es.pwd" $ELASTICSEARCH_PASSWORD buildkite-agent meta-data set "kibana.host" $KIBANA_HOST buildkite-agent meta-data set "kibana.username" $KIBANA_USERNAME buildkite-agent meta-data set "kibana.pwd" $KIBANA_PASSWORD +buildkite-agent meta-data set "integrations_server.host" $INTEGRATIONS_SERVER_HOST diff --git a/test_infra/ess/output.tf b/test_infra/ess/output.tf index 4c6427e0f22..16bf7bf369c 100644 --- a/test_infra/ess/output.tf +++ b/test_infra/ess/output.tf @@ -23,3 +23,7 @@ output "es_host" { output "kibana_endpoint" { value = ec_deployment.integration-testing.kibana.https_endpoint } + +output "integrations_server_endpoint" { + value = ec_deployment.integration-testing.integrations_server.https_endpoint +} diff --git a/testing/integration/fleetserver_fips_test.go b/testing/integration/fleetserver_fips_test.go index 9157ba29a3f..5e1e467f19c 100644 --- a/testing/integration/fleetserver_fips_test.go +++ b/testing/integration/fleetserver_fips_test.go @@ -42,7 +42,7 @@ func TestFIPSAgentConnectingToFIPSFleetServerInECHFRH(t *testing.T) { }) // Check that the Fleet Server in the deployment is healthy - fleetServerHost := os.Getenv("FLEETSERVER_HOST") + fleetServerHost := os.Getenv("INTEGRATIONS_SERVER_HOST") statusUrl, err := url.JoinPath(fleetServerHost, "/api/status") require.NoError(t, err) From a0f7d75c6fcf3b3bf2d5c545019fd515de4de467 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 9 May 2025 17:11:36 -0700 Subject: [PATCH 12/14] Set up INTEGRATIONS_SERVER_HOST environment variable --- .../scripts/steps/integration_tests_tf.sh | 1 + docs/test-framework-dev-guide.md | 3 ++- magefile.go | 3 +++ pkg/testing/common/stack.go | 4 ++++ pkg/testing/ess/client_test.go | 1 + pkg/testing/ess/deployment.go | 13 +++++++++++-- pkg/testing/ess/statful_provisioner.go | 15 ++++++++------- pkg/testing/runner/runner.go | 2 ++ pkg/testing/runner/runner_test.go | 17 +++++++++-------- 9 files changed, 41 insertions(+), 18 deletions(-) diff --git a/.buildkite/scripts/steps/integration_tests_tf.sh b/.buildkite/scripts/steps/integration_tests_tf.sh index ab51a46e2a1..c4d2fd02819 100755 --- a/.buildkite/scripts/steps/integration_tests_tf.sh +++ b/.buildkite/scripts/steps/integration_tests_tf.sh @@ -47,6 +47,7 @@ else export KIBANA_HOST=$(buildkite-agent meta-data get "kibana.host") export KIBANA_USERNAME=$(buildkite-agent meta-data get "kibana.username") export KIBANA_PASSWORD=$(buildkite-agent meta-data get "kibana.pwd") + export INTEGRATIONS_SERVER_HOST=$(buildkite-agent meta-data get "integrations_server.host") fi # TODO: move to common.sh when it's refactored diff --git a/docs/test-framework-dev-guide.md b/docs/test-framework-dev-guide.md index b2558f17b9d..f2e5df937a9 100644 --- a/docs/test-framework-dev-guide.md +++ b/docs/test-framework-dev-guide.md @@ -148,7 +148,8 @@ following `mage integration:*` commands to re-use the already provisioned resour Tests with external dependencies might need more environment variables to be set when running them manually, such as `ELASTICSEARCH_HOST`, `ELASTICSEARCH_USERNAME`, -`ELASTICSEARCH_PASSWORD`, `KIBANA_HOST`, `KIBANA_USERNAME`, and `KIBANA_PASSWORD`. +`ELASTICSEARCH_PASSWORD`, `KIBANA_HOST`, `KIBANA_USERNAME`, `KIBANA_PASSWORD`, and +`INTEGRATIONS_SERVER_HOST`. ### Debugging tests diff --git a/magefile.go b/magefile.go index 72c787e72bb..efce4501627 100644 --- a/magefile.go +++ b/magefile.go @@ -2486,6 +2486,7 @@ func listStacks() (string, error) { t.AppendRows([]table.Row{ {"Elasticsearch URL", stack.Elasticsearch}, {"Kibana", stack.Kibana}, + {"Integrations Server", stack.IntegrationsServer}, {"Username", stack.Username}, {"Password", stack.Password}, }) @@ -2579,6 +2580,8 @@ func generateEnvFile(stack tcommon.Stack) error { fmt.Fprintf(f, "export KIBANA_USERNAME=\"%s\"\n", stack.Username) fmt.Fprintf(f, "export KIBANA_PASSWORD=\"%s\"\n", stack.Password) + fmt.Fprintf(f, "export INTEGRATIONS_SERVER_HOST=\"%s\"\n", stack.IntegrationsServer) + return nil } diff --git a/pkg/testing/common/stack.go b/pkg/testing/common/stack.go index 3047b340ea0..e204840a623 100644 --- a/pkg/testing/common/stack.go +++ b/pkg/testing/common/stack.go @@ -29,6 +29,10 @@ type Stack struct { // Kibana is the URL to communication with kibana. Kibana string `yaml:"kibana"` + // IntegrationsServer is the URL to communicate with integrations server + // (i.e. fleet server). + IntegrationsServer string `yaml:"integrations_server"` + // Username is the username. Username string `yaml:"username"` diff --git a/pkg/testing/ess/client_test.go b/pkg/testing/ess/client_test.go index fe158fc3d80..a19e2a6b865 100644 --- a/pkg/testing/ess/client_test.go +++ b/pkg/testing/ess/client_test.go @@ -35,6 +35,7 @@ func TestClient_CreateAndShutdownDeployment(t *testing.T) { require.NotEmpty(t, resp.ID) require.NotEmpty(t, resp.ElasticsearchEndpoint) require.NotEmpty(t, resp.KibanaEndpoint) + require.NotEmpty(t, resp.IntegrationsServerEndpoint) require.NotEmpty(t, resp.Username) require.NotEmpty(t, resp.Password) diff --git a/pkg/testing/ess/deployment.go b/pkg/testing/ess/deployment.go index 3dc2474bdf3..4870495acf9 100644 --- a/pkg/testing/ess/deployment.go +++ b/pkg/testing/ess/deployment.go @@ -36,8 +36,9 @@ type CreateDeploymentRequest struct { type CreateDeploymentResponse struct { ID string `json:"id"` - ElasticsearchEndpoint string - KibanaEndpoint string + ElasticsearchEndpoint string + KibanaEndpoint string + IntegrationsServerEndpoint string Username string Password string @@ -170,6 +171,13 @@ func (c *Client) CreateDeployment(ctx context.Context, req CreateDeploymentReque } `json:"metadata"` } `json:"info"` } `json:"kibana"` + IntegrationsServer []struct { + Info struct { + Metadata struct { + ServiceUrl string `json:"service_url"` + } `json:"metadata"` + } `json:"info"` + } `json:"integrations_server"` } `json:"resources"` } @@ -179,6 +187,7 @@ func (c *Client) CreateDeployment(ctx context.Context, req CreateDeploymentReque r.ElasticsearchEndpoint = getRespBody.Resources.Elasticsearch[0].Info.Metadata.ServiceUrl r.KibanaEndpoint = getRespBody.Resources.Kibana[0].Info.Metadata.ServiceUrl + r.IntegrationsServerEndpoint = getRespBody.Resources.IntegrationsServer[0].Info.Metadata.ServiceUrl return &r, nil } diff --git a/pkg/testing/ess/statful_provisioner.go b/pkg/testing/ess/statful_provisioner.go index 7c6d79d5f9f..4a30cfd2f54 100644 --- a/pkg/testing/ess/statful_provisioner.go +++ b/pkg/testing/ess/statful_provisioner.go @@ -91,13 +91,14 @@ func (p *statefulProvisioner) Create(ctx context.Context, request common.StackRe return common.Stack{}, err } return common.Stack{ - ID: request.ID, - Provisioner: p.Name(), - Version: request.Version, - Elasticsearch: resp.ElasticsearchEndpoint, - Kibana: resp.KibanaEndpoint, - Username: resp.Username, - Password: resp.Password, + ID: request.ID, + Provisioner: p.Name(), + Version: request.Version, + Elasticsearch: resp.ElasticsearchEndpoint, + Kibana: resp.KibanaEndpoint, + IntegrationsServer: resp.IntegrationsServerEndpoint, + Username: resp.Username, + Password: resp.Password, Internal: map[string]interface{}{ "deployment_id": resp.ID, }, diff --git a/pkg/testing/runner/runner.go b/pkg/testing/runner/runner.go index 973df39f41f..96d98fc13bc 100644 --- a/pkg/testing/runner/runner.go +++ b/pkg/testing/runner/runner.go @@ -261,6 +261,7 @@ func (r *Runner) runK8sInstances(ctx context.Context, instances []StateInstance) env["KIBANA_HOST"] = stack.Kibana env["KIBANA_USERNAME"] = stack.Username env["KIBANA_PASSWORD"] = stack.Password + env["INTEGRATIONS_SERVER_HOST"] = stack.IntegrationsServer logger.Logf("Using Stack with Kibana host %s, credentials available under .integration-cache", stack.Kibana) } @@ -384,6 +385,7 @@ func (r *Runner) runInstance(ctx context.Context, sshAuth ssh.AuthMethod, logger env["KIBANA_HOST"] = stack.Kibana env["KIBANA_USERNAME"] = stack.Username env["KIBANA_PASSWORD"] = stack.Password + env["INTEGRATIONS_SERVER_HOST"] = stack.IntegrationsServer logger.Logf("Using Stack with Kibana host %s, credentials available under .integration-cache", stack.Kibana) } diff --git a/pkg/testing/runner/runner_test.go b/pkg/testing/runner/runner_test.go index 6fb271bdb89..8a3dee64937 100644 --- a/pkg/testing/runner/runner_test.go +++ b/pkg/testing/runner/runner_test.go @@ -158,14 +158,15 @@ func (p *fakeStackProvisioner) Create(_ context.Context, request common.StackReq defer p.mx.Unlock() p.requests = append(p.requests, request) return common.Stack{ - ID: request.ID, - Version: request.Version, - Elasticsearch: "http://localhost:9200", - Kibana: "http://localhost:5601", - Username: "elastic", - Password: "changeme", - Internal: nil, - Ready: false, + ID: request.ID, + Version: request.Version, + Elasticsearch: "http://localhost:9200", + Kibana: "http://localhost:5601", + IntegrationsServer: "http://localhost:8220", + Username: "elastic", + Password: "changeme", + Internal: nil, + Ready: false, }, nil } From 741178b2307a4137febe858c7e02e408b117eb19 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 9 May 2025 23:35:03 -0700 Subject: [PATCH 13/14] Fixing API errors --- testing/integration/fleetserver_fips_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/testing/integration/fleetserver_fips_test.go b/testing/integration/fleetserver_fips_test.go index 5e1e467f19c..51a361f2f3c 100644 --- a/testing/integration/fleetserver_fips_test.go +++ b/testing/integration/fleetserver_fips_test.go @@ -17,6 +17,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/elastic/elastic-agent-libs/kibana" "github.com/elastic/elastic-agent/pkg/testing/define" ) @@ -61,9 +62,9 @@ func TestFIPSAgentConnectingToFIPSFleetServerInECHFRH(t *testing.T) { require.Equal(t, "HEALTHY", body.Status) // Get all Agents - ctx, cancel := context.WithTimeout(5*time.Second, context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - agents, err := info.KibanaClient.ListAgents(ctx) + agents, err := info.KibanaClient.ListAgents(ctx, kibana.ListAgentsRequest{}) require.NoError(t, err) // Find Fleet Server's own Agent and get its status and whether it's From c0072c84acab4ffa0f45f5c6632f916980d06e85 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 13 May 2025 04:39:46 -0700 Subject: [PATCH 14/14] Bump elastic-agent-libs --- NOTICE-fips.txt | 4 ++-- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- testing/integration/fleetserver_fips_test.go | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/NOTICE-fips.txt b/NOTICE-fips.txt index eedb417b513..106e83f71d0 100644 --- a/NOTICE-fips.txt +++ b/NOTICE-fips.txt @@ -1224,11 +1224,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.19.4 +Version: v0.19.5 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.19.4/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.19.5/LICENSE: Apache License Version 2.0, January 2004 diff --git a/NOTICE.txt b/NOTICE.txt index d14071961c4..20b590c812c 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1224,11 +1224,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.19.4 +Version: v0.19.5 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.19.4/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.19.5/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index c5d4584710c..cbbd184ce28 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/elastic/cloud-on-k8s/v2 v2.0.0-20250327073047-b624240832ae github.com/elastic/elastic-agent-autodiscover v0.9.0 github.com/elastic/elastic-agent-client/v7 v7.17.2 - github.com/elastic/elastic-agent-libs v0.19.4 + github.com/elastic/elastic-agent-libs v0.19.5 github.com/elastic/elastic-agent-system-metrics v0.11.12 github.com/elastic/elastic-transport-go/v8 v8.7.0 github.com/elastic/go-elasticsearch/v8 v8.17.1 diff --git a/go.sum b/go.sum index 712694e547c..4056c864f12 100644 --- a/go.sum +++ b/go.sum @@ -482,8 +482,8 @@ github.com/elastic/elastic-agent-autodiscover v0.9.0 h1:+iWIKh0u3e8I+CJa3FfWe9h0 github.com/elastic/elastic-agent-autodiscover v0.9.0/go.mod h1:5iUxLHhVdaGSWYTveSwfJEY4RqPXTG13LPiFoxcpFd4= github.com/elastic/elastic-agent-client/v7 v7.17.2 h1:Cl2TeABqWZgW40t5fchGWT/sRk4MDDLWA0d8iHHOxLA= github.com/elastic/elastic-agent-client/v7 v7.17.2/go.mod h1:5irRFqp6HLqtu1S+OeY0jg8x7K6PLL+DW+PwVk1vJnk= -github.com/elastic/elastic-agent-libs v0.19.4 h1:ydeIBvNmfvJKwmmRQKAad4R9KcRsmYaPfzw9pGYu5+w= -github.com/elastic/elastic-agent-libs v0.19.4/go.mod h1:1HNxREH8C27kGrJCtKZh/ot8pV8joH8VREP21+FrH5s= +github.com/elastic/elastic-agent-libs v0.19.5 h1:br0kvmwowHu4KQpPy5C4DjkFdiGafbp8BV4CeXR4wUs= +github.com/elastic/elastic-agent-libs v0.19.5/go.mod h1:1HNxREH8C27kGrJCtKZh/ot8pV8joH8VREP21+FrH5s= github.com/elastic/elastic-agent-system-metrics v0.11.12 h1:2SnN3arw2MrWV7eWjm+8zLuyQcsCtEyMasi1w9eFHU8= github.com/elastic/elastic-agent-system-metrics v0.11.12/go.mod h1:GNqmKfvOt8PwORjbS6GllNdMfkLpOWyTa7P8oQq4E5o= github.com/elastic/elastic-transport-go/v8 v8.7.0 h1:OgTneVuXP2uip4BA658Xi6Hfw+PeIOod2rY3GVMGoVE= diff --git a/testing/integration/fleetserver_fips_test.go b/testing/integration/fleetserver_fips_test.go index 51a361f2f3c..4b8be5d33b2 100644 --- a/testing/integration/fleetserver_fips_test.go +++ b/testing/integration/fleetserver_fips_test.go @@ -74,7 +74,7 @@ func TestFIPSAgentConnectingToFIPSFleetServerInECHFRH(t *testing.T) { for _, item := range agents.Items { if item.PolicyID == cloudAgentPolicyID { agentStatus = item.Status - //agentIsFIPS = item.LocalMetadata.Elastic.Agent.FIPS + agentIsFIPS = item.LocalMetadata.Elastic.Agent.FIPS } }