From 5b2007a0afdc2d6d23094f540da55595c662ebda Mon Sep 17 00:00:00 2001 From: Toby Brain Date: Thu, 18 Sep 2025 12:40:39 +1000 Subject: [PATCH 1/2] Support secret ref arrays in integration policies --- .../fleet/integration_policy/resource_test.go | 170 +++++++++++++----- internal/fleet/integration_policy/secrets.go | 36 +++- .../fleet/integration_policy/secrets_test.go | 33 +++- 3 files changed, 180 insertions(+), 59 deletions(-) diff --git a/internal/fleet/integration_policy/resource_test.go b/internal/fleet/integration_policy/resource_test.go index 70c9375d3..502830ac5 100644 --- a/internal/fleet/integration_policy/resource_test.go +++ b/internal/fleet/integration_policy/resource_test.go @@ -20,7 +20,10 @@ import ( "github.com/stretchr/testify/require" ) -var minVersionIntegrationPolicy = version.Must(version.NewVersion("8.10.0")) +var ( + minVersionIntegrationPolicy = version.Must(version.NewVersion("8.10.0")) + minVersionSqlIntegration = version.Must(version.NewVersion("9.1.0")) +) func TestJsonTypes(t *testing.T) { mapBytes, err := json.Marshal(map[string]string{}) @@ -127,53 +130,92 @@ func TestAccResourceIntegrationPolicySecretsFromSDK(t *testing.T) { func TestAccResourceIntegrationPolicySecrets(t *testing.T) { policyName := sdkacctest.RandStringFromCharSet(22, sdkacctest.CharSetAlphaNum) - resource.Test(t, resource.TestCase{ - PreCheck: func() { acctest.PreCheck(t) }, - CheckDestroy: checkResourceIntegrationPolicyDestroy, - ProtoV6ProviderFactories: acctest.Providers, - Steps: []resource.TestStep{ - { - SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy), - Config: testAccResourceIntegrationPolicySecretsCreate(policyName, "created"), - Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "name", policyName), - resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "description", "IntegrationPolicyTest Policy"), - resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_name", "aws_logs"), - resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_version", "1.4.0"), - resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", fmt.Sprintf(`{"access_key_id":"placeholder","default_region":"us-east-1","endpoint":"endpoint","secret_access_key":"created %s","session_token":"placeholder"}`, policyName)), - resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.input_id", "aws_logs-aws-cloudwatch"), - resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.enabled", "true"), - resource.TestCheckNoResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.vars_json"), - resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", `{"aws_logs.generic":{"enabled":true,"vars":{"api_sleep":"200ms","api_timeput":"120s","custom":"","data_stream.dataset":"aws_logs.generic","log_streams":[],"number_of_workers":1,"preserve_original_event":false,"scan_frequency":"1m","start_position":"beginning","tags":["forwarded"]}}}`), - ), - }, - { - SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy), - Config: testAccResourceIntegrationPolicySecretsUpdate(policyName, "updated"), - Check: resource.ComposeTestCheckFunc( - resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "name", policyName), - resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "description", "Updated Integration Policy"), - resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_name", "aws_logs"), - resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_version", "1.4.0"), - resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", fmt.Sprintf(`{"access_key_id":"placeholder","default_region":"us-east-2","endpoint":"endpoint","secret_access_key":"updated %s","session_token":"placeholder"}`, policyName)), - resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.input_id", "aws_logs-aws-cloudwatch"), - resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.enabled", "false"), - resource.TestCheckNoResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.vars_json"), - resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", `{"aws_logs.generic":{"enabled":false,"vars":{"api_sleep":"200ms","api_timeput":"120s","custom":"","data_stream.dataset":"aws_logs.generic","log_streams":[],"number_of_workers":1,"preserve_original_event":false,"scan_frequency":"2m","start_position":"beginning","tags":["forwarded"]}}}`), - ), + t.Run("single valued secrets", func(t *testing.T) { + resource.Test(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(t) }, + CheckDestroy: checkResourceIntegrationPolicyDestroy, + ProtoV6ProviderFactories: acctest.Providers, + Steps: []resource.TestStep{ + { + SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy), + Config: testAccResourceIntegrationPolicySecretsCreate(policyName, "created"), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "name", policyName), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "description", "IntegrationPolicyTest Policy"), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_name", "aws_logs"), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_version", "1.4.0"), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", fmt.Sprintf(`{"access_key_id":"placeholder","default_region":"us-east-1","endpoint":"endpoint","secret_access_key":"created %s","session_token":"placeholder"}`, policyName)), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.input_id", "aws_logs-aws-cloudwatch"), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.enabled", "true"), + resource.TestCheckNoResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.vars_json"), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", `{"aws_logs.generic":{"enabled":true,"vars":{"api_sleep":"200ms","api_timeput":"120s","custom":"","data_stream.dataset":"aws_logs.generic","log_streams":[],"number_of_workers":1,"preserve_original_event":false,"scan_frequency":"1m","start_position":"beginning","tags":["forwarded"]}}}`), + ), + }, + { + SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy), + Config: testAccResourceIntegrationPolicySecretsUpdate(policyName, "updated"), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "name", policyName), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "description", "Updated Integration Policy"), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_name", "aws_logs"), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_version", "1.4.0"), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", fmt.Sprintf(`{"access_key_id":"placeholder","default_region":"us-east-2","endpoint":"endpoint","secret_access_key":"updated %s","session_token":"placeholder"}`, policyName)), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.input_id", "aws_logs-aws-cloudwatch"), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.enabled", "false"), + resource.TestCheckNoResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.vars_json"), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", `{"aws_logs.generic":{"enabled":false,"vars":{"api_sleep":"200ms","api_timeput":"120s","custom":"","data_stream.dataset":"aws_logs.generic","log_streams":[],"number_of_workers":1,"preserve_original_event":false,"scan_frequency":"2m","start_position":"beginning","tags":["forwarded"]}}}`), + ), + }, + { + SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy), + ResourceName: "elasticstack_fleet_integration_policy.test_policy", + Config: testAccResourceIntegrationPolicyUpdate(policyName), + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"vars_json"}, + Check: resource.ComposeTestCheckFunc( + resource.TestMatchResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", regexp.MustCompile(`{"access_key_id":{"id":"\S+","isSecretRef":true},"default_region":"us-east-2","endpoint":"endpoint","secret_access_key":{"id":"\S+","isSecretRef":true},"session_token":{"id":"\S+","isSecretRef":true}}`)), + ), + }, }, - { - SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy), - ResourceName: "elasticstack_fleet_integration_policy.test_policy", - Config: testAccResourceIntegrationPolicyUpdate(policyName), - ImportState: true, - ImportStateVerify: true, - ImportStateVerifyIgnore: []string{"vars_json"}, - Check: resource.ComposeTestCheckFunc( - resource.TestMatchResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", regexp.MustCompile(`{"access_key_id":{"id":"\S+","isSecretRef":true},"default_region":"us-east-2","endpoint":"endpoint","secret_access_key":{"id":"\S+","isSecretRef":true},"session_token":{"id":"\S+","isSecretRef":true}}`)), - ), + }) + }) + + t.Run("multi-valued secrets", func(t *testing.T) { + resource.Test(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(t) }, + CheckDestroy: checkResourceIntegrationPolicyDestroy, + ProtoV6ProviderFactories: acctest.Providers, + Steps: []resource.TestStep{ + { + SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionSqlIntegration), + Config: testAccResourceIntegrationPolicySecretsIds(policyName, "created"), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "name", policyName), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "description", "SQL Integration Policy"), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_name", "sql"), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_version", "1.1.0"), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.input_id", "sql-sql/metrics"), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.enabled", "true"), + resource.TestCheckNoResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.vars_json"), + resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", `{"sql.sql":{"enabled":true,"vars":{"data_stream.dataset":"sql","driver":"mysql","hosts":["root:test@tcp(127.0.0.1:3306)/"],"merge_results":false,"period":"1m","processors":"","sql_queries":"- query: SHOW GLOBAL STATUS LIKE 'Innodb_system%'\n response_format: variables\n \n","ssl":""}}}`), + ), + }, + { + SkipFunc: func() (bool, error) { + return versionutils.CheckIfVersionIsUnsupported(minVersionSqlIntegration)() + }, + ResourceName: "elasticstack_fleet_integration_policy.test_policy", + Config: testAccResourceIntegrationPolicySecretsIds(policyName, "created"), + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"input.0.streams_json"}, + Check: resource.ComposeTestCheckFunc( + resource.TestMatchResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", regexp.MustCompile(`"hosts":{"ids":["\S+"],"isSecretRef":true}`)), + ), + }, }, - }, + }) }) } @@ -451,3 +493,39 @@ resource "elasticstack_fleet_integration_policy" "test_policy" { } `, common, id, key, id) } + +func testAccResourceIntegrationPolicySecretsIds(id string, key string) string { + common := testAccResourceIntegrationPolicyCommon(id, "sql", "1.1.0") + return fmt.Sprintf(` +%s + +resource "elasticstack_fleet_integration_policy" "test_policy" { + name = "%s" + namespace = "default" + description = "SQL Integration Policy" + agent_policy_id = elasticstack_fleet_agent_policy.test_policy.policy_id + integration_name = elasticstack_fleet_integration.test_policy.name + integration_version = elasticstack_fleet_integration.test_policy.version + + input { + input_id = "sql-sql/metrics" + enabled = true + streams_json = jsonencode({ + "sql.sql" : { + "enabled" : true, + "vars" : { + "hosts" : ["root:test@tcp(127.0.0.1:3306)/"], + "period" : "1m", + "driver" : "mysql", + "sql_queries" : "- query: SHOW GLOBAL STATUS LIKE 'Innodb_system%%'\n response_format: variables\n \n", + "merge_results" : false, + "ssl" : "", + "data_stream.dataset" : "sql", + "processors" : "" + } + } + }) + } +} +`, common, id) +} diff --git a/internal/fleet/integration_policy/secrets.go b/internal/fleet/integration_policy/secrets.go index 0a5ef36a8..0423a3798 100644 --- a/internal/fleet/integration_policy/secrets.go +++ b/internal/fleet/integration_policy/secrets.go @@ -71,9 +71,21 @@ func HandleRespSecrets(ctx context.Context, resp *kbapi.PackagePolicy, private p } handleVar := func(key string, mval map[string]any, vars map[string]any) { - refID := mval["id"].(string) - if original, ok := secrets[refID]; ok { - vars[key] = original + if refID, ok := mval["id"]; ok { + if original, ok := secrets[refID.(string)]; ok { + vars[key] = original + } + } else if ids, ok := mval["ids"]; ok { + values := []any{} + for _, id := range ids.([]any) { + if original, ok := secrets[id.(string)]; ok { + values = append(values, original) + } + } + + if len(values) > 0 { + vars[key] = values + } } } @@ -136,8 +148,22 @@ func HandleReqRespSecrets(ctx context.Context, req kbapi.PackagePolicyRequest, r } } - refID := mval["id"].(string) - secrets[refID] = original + if refID, ok := mval["id"]; ok { + secrets[refID.(string)] = original + } else if ids, ok := mval["ids"]; ok { + originals, ok := original.([]any) + if !ok || len(originals) != len(ids.([]any)) { + diags.AddError("mismatched secret ref ids and original values", "the number of secret ref ids does not match the number of original values") + return + } + + // Map each id to the corresponding original value by position. + // The API does not return the original value with the id, + // so we have to assume the order is preserved. + for i, id := range ids.([]any) { + secrets[id.(string)] = originals[i] + } + } } } diff --git a/internal/fleet/integration_policy/secrets_test.go b/internal/fleet/integration_policy/secrets_test.go index c62348c00..811ce25e6 100644 --- a/internal/fleet/integration_policy/secrets_test.go +++ b/internal/fleet/integration_policy/secrets_test.go @@ -33,10 +33,12 @@ func TestHandleRespSecrets(t *testing.T) { t.Parallel() ctx := context.Background() - private := privateData{"secrets": `{"known-secret":"secret"}`} + private := privateData{"secrets": `{"known-secret":"secret", "known-secret-1":"secret1", "known-secret-2":"secret2"}`} secretRefs := &[]kbapi.PackagePolicySecretRef{ {Id: "known-secret"}, + {Id: "known-secret-1"}, + {Id: "known-secret-2"}, } tests := []struct { @@ -64,6 +66,11 @@ func TestHandleRespSecrets(t *testing.T) { input: Map{"k": Map{"isSecretRef": true, "id": "known-secret"}}, want: Map{"k": "secret"}, }, + { + name: "converts secret with multiple values", + input: Map{"k": Map{"isSecretRef": true, "ids": []any{"known-secret-1", "known-secret-2"}}}, + want: Map{"k": []any{"secret1", "secret2"}}, + }, { name: "converts wrapped secret", input: Map{"k": Map{"type": "password", "value": Map{"isSecretRef": true, "id": "known-secret"}}}, @@ -121,7 +128,7 @@ func TestHandleRespSecrets(t *testing.T) { require.Equal(t, want, got) // privateData - privateWants := privateData{"secrets": `{"known-secret":"secret"}`} + privateWants := privateData{"secrets": `{"known-secret":"secret","known-secret-1":"secret1","known-secret-2":"secret2"}`} require.Equal(t, privateWants, private) }) } @@ -134,6 +141,8 @@ func TestHandleReqRespSecrets(t *testing.T) { secretRefs := &[]kbapi.PackagePolicySecretRef{ {Id: "known-secret"}, + {Id: "known-secret-1"}, + {Id: "known-secret-2"}, } tests := []struct { @@ -166,6 +175,12 @@ func TestHandleReqRespSecrets(t *testing.T) { respInput: Map{"k": Map{"isSecretRef": true, "id": "known-secret"}}, want: Map{"k": "secret"}, }, + { + name: "converts secret with multiple values", + reqInput: Map{"k": []any{"secret1", "secret2"}}, + respInput: Map{"k": Map{"isSecretRef": true, "ids": []any{"known-secret-1", "known-secret-2"}}}, + want: Map{"k": []any{"secret1", "secret2"}}, + }, { name: "converts wrapped secret", reqInput: Map{"k": "secret"}, @@ -236,13 +251,15 @@ func TestHandleReqRespSecrets(t *testing.T) { want = *(*wants.Inputs["input1"].Streams)["stream1"].Vars require.Equal(t, want, got) - if v, ok := (*req.Vars)["k"]; ok && v == "secret" { - privateWants := privateData{"secrets": `{"known-secret":"secret"}`} - require.Equal(t, privateWants, private) - } else { - privateWants := privateData{"secrets": `{}`} - require.Equal(t, privateWants, private) + privateWants := privateData{"secrets": `{}`} + if v, ok := (*req.Vars)["k"]; ok { + if s, ok := v.(string); ok && s == "secret" { + privateWants = privateData{"secrets": `{"known-secret":"secret"}`} + } else if _, ok := v.([]any); ok { + privateWants = privateData{"secrets": `{"known-secret-1":"secret1","known-secret-2":"secret2"}`} + } } + require.Equal(t, privateWants, private) }) } } From 27fc33bd35363cacf6ad7cad8382635bb09c5eb7 Mon Sep 17 00:00:00 2001 From: Toby Brain Date: Thu, 18 Sep 2025 21:28:43 +1000 Subject: [PATCH 2/2] Bump the highest stack version to 9.1.3 --- .github/workflows/test.yml | 8 +++++--- Makefile | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4d2eec47b..2c7aae06c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,9 +125,11 @@ jobs: - '8.14.3' - '8.15.5' - '8.16.2' - - '8.17.0' - - '8.18.3' - - '9.0.3' + - '8.17.10' + - '8.18.7' + - '8.19.3' + - '9.0.7' + - '9.1.3' steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 - uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6 diff --git a/Makefile b/Makefile index d94bcf925..51d535ce7 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ SWAGGER_VERSION ?= 8.7 GOVERSION ?= $(shell grep -e '^go' go.mod | cut -f 2 -d ' ') -STACK_VERSION ?= 9.0.3 +STACK_VERSION ?= 9.1.3 ELASTICSEARCH_NAME ?= terraform-elasticstack-es ELASTICSEARCH_ENDPOINTS ?= http://$(ELASTICSEARCH_NAME):9200