Skip to content

Support secret ref arrays in integration policies #1316

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 27, 2025
Merged

Support secret ref arrays in integration policies #1316

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5b2007a
Support secret ref arrays in integration policies
tobio Sep 18, 2025
27fc33b
Bump the highest stack version to 9.1.3
tobio Sep 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,11 @@ jobs:
- '8.14.3'
- '8.15.5'
- '8.16.2'
- '8.17.0'
- '8.18.3'
- '9.0.3'
- '8.17.10'
- '8.18.7'
- '8.19.3'
- '9.0.7'
- '9.1.3'
steps:
- uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
- uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6
Expand Down
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ SWAGGER_VERSION ?= 8.7

GOVERSION ?= $(shell grep -e '^go' go.mod | cut -f 2 -d ' ')

STACK_VERSION ?= 9.0.3
STACK_VERSION ?= 9.1.3

ELASTICSEARCH_NAME ?= terraform-elasticstack-es
ELASTICSEARCH_ENDPOINTS ?= http://$(ELASTICSEARCH_NAME):9200
Expand Down
170 changes: 124 additions & 46 deletions internal/fleet/integration_policy/resource_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,10 @@ import (
"github.com/stretchr/testify/require"
)

var minVersionIntegrationPolicy = version.Must(version.NewVersion("8.10.0"))
var (
minVersionIntegrationPolicy = version.Must(version.NewVersion("8.10.0"))
minVersionSqlIntegration = version.Must(version.NewVersion("9.1.0"))
)

func TestJsonTypes(t *testing.T) {
mapBytes, err := json.Marshal(map[string]string{})
Expand Down Expand Up @@ -127,53 +130,92 @@ func TestAccResourceIntegrationPolicySecretsFromSDK(t *testing.T) {
func TestAccResourceIntegrationPolicySecrets(t *testing.T) {
policyName := sdkacctest.RandStringFromCharSet(22, sdkacctest.CharSetAlphaNum)

resource.Test(t, resource.TestCase{
PreCheck: func() { acctest.PreCheck(t) },
CheckDestroy: checkResourceIntegrationPolicyDestroy,
ProtoV6ProviderFactories: acctest.Providers,
Steps: []resource.TestStep{
{
SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy),
Config: testAccResourceIntegrationPolicySecretsCreate(policyName, "created"),
Check: resource.ComposeTestCheckFunc(
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "name", policyName),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "description", "IntegrationPolicyTest Policy"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_name", "aws_logs"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_version", "1.4.0"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", fmt.Sprintf(`{"access_key_id":"placeholder","default_region":"us-east-1","endpoint":"endpoint","secret_access_key":"created %s","session_token":"placeholder"}`, policyName)),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.input_id", "aws_logs-aws-cloudwatch"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.enabled", "true"),
resource.TestCheckNoResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.vars_json"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", `{"aws_logs.generic":{"enabled":true,"vars":{"api_sleep":"200ms","api_timeput":"120s","custom":"","data_stream.dataset":"aws_logs.generic","log_streams":[],"number_of_workers":1,"preserve_original_event":false,"scan_frequency":"1m","start_position":"beginning","tags":["forwarded"]}}}`),
),
},
{
SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy),
Config: testAccResourceIntegrationPolicySecretsUpdate(policyName, "updated"),
Check: resource.ComposeTestCheckFunc(
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "name", policyName),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "description", "Updated Integration Policy"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_name", "aws_logs"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_version", "1.4.0"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", fmt.Sprintf(`{"access_key_id":"placeholder","default_region":"us-east-2","endpoint":"endpoint","secret_access_key":"updated %s","session_token":"placeholder"}`, policyName)),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.input_id", "aws_logs-aws-cloudwatch"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.enabled", "false"),
resource.TestCheckNoResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.vars_json"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", `{"aws_logs.generic":{"enabled":false,"vars":{"api_sleep":"200ms","api_timeput":"120s","custom":"","data_stream.dataset":"aws_logs.generic","log_streams":[],"number_of_workers":1,"preserve_original_event":false,"scan_frequency":"2m","start_position":"beginning","tags":["forwarded"]}}}`),
),
t.Run("single valued secrets", func(t *testing.T) {
resource.Test(t, resource.TestCase{
PreCheck: func() { acctest.PreCheck(t) },
CheckDestroy: checkResourceIntegrationPolicyDestroy,
ProtoV6ProviderFactories: acctest.Providers,
Steps: []resource.TestStep{
{
SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy),
Config: testAccResourceIntegrationPolicySecretsCreate(policyName, "created"),
Check: resource.ComposeTestCheckFunc(
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "name", policyName),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "description", "IntegrationPolicyTest Policy"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_name", "aws_logs"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_version", "1.4.0"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", fmt.Sprintf(`{"access_key_id":"placeholder","default_region":"us-east-1","endpoint":"endpoint","secret_access_key":"created %s","session_token":"placeholder"}`, policyName)),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.input_id", "aws_logs-aws-cloudwatch"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.enabled", "true"),
resource.TestCheckNoResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.vars_json"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", `{"aws_logs.generic":{"enabled":true,"vars":{"api_sleep":"200ms","api_timeput":"120s","custom":"","data_stream.dataset":"aws_logs.generic","log_streams":[],"number_of_workers":1,"preserve_original_event":false,"scan_frequency":"1m","start_position":"beginning","tags":["forwarded"]}}}`),
),
},
{
SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy),
Config: testAccResourceIntegrationPolicySecretsUpdate(policyName, "updated"),
Check: resource.ComposeTestCheckFunc(
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "name", policyName),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "description", "Updated Integration Policy"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_name", "aws_logs"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_version", "1.4.0"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", fmt.Sprintf(`{"access_key_id":"placeholder","default_region":"us-east-2","endpoint":"endpoint","secret_access_key":"updated %s","session_token":"placeholder"}`, policyName)),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.input_id", "aws_logs-aws-cloudwatch"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.enabled", "false"),
resource.TestCheckNoResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.vars_json"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", `{"aws_logs.generic":{"enabled":false,"vars":{"api_sleep":"200ms","api_timeput":"120s","custom":"","data_stream.dataset":"aws_logs.generic","log_streams":[],"number_of_workers":1,"preserve_original_event":false,"scan_frequency":"2m","start_position":"beginning","tags":["forwarded"]}}}`),
),
},
{
SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy),
ResourceName: "elasticstack_fleet_integration_policy.test_policy",
Config: testAccResourceIntegrationPolicyUpdate(policyName),
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"vars_json"},
Check: resource.ComposeTestCheckFunc(
resource.TestMatchResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", regexp.MustCompile(`{"access_key_id":{"id":"\S+","isSecretRef":true},"default_region":"us-east-2","endpoint":"endpoint","secret_access_key":{"id":"\S+","isSecretRef":true},"session_token":{"id":"\S+","isSecretRef":true}}`)),
),
},
},
{
SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionIntegrationPolicy),
ResourceName: "elasticstack_fleet_integration_policy.test_policy",
Config: testAccResourceIntegrationPolicyUpdate(policyName),
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"vars_json"},
Check: resource.ComposeTestCheckFunc(
resource.TestMatchResourceAttr("elasticstack_fleet_integration_policy.test_policy", "vars_json", regexp.MustCompile(`{"access_key_id":{"id":"\S+","isSecretRef":true},"default_region":"us-east-2","endpoint":"endpoint","secret_access_key":{"id":"\S+","isSecretRef":true},"session_token":{"id":"\S+","isSecretRef":true}}`)),
),
})
})

t.Run("multi-valued secrets", func(t *testing.T) {
resource.Test(t, resource.TestCase{
PreCheck: func() { acctest.PreCheck(t) },
CheckDestroy: checkResourceIntegrationPolicyDestroy,
ProtoV6ProviderFactories: acctest.Providers,
Steps: []resource.TestStep{
{
SkipFunc: versionutils.CheckIfVersionIsUnsupported(minVersionSqlIntegration),
Config: testAccResourceIntegrationPolicySecretsIds(policyName, "created"),
Check: resource.ComposeTestCheckFunc(
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "name", policyName),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "description", "SQL Integration Policy"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_name", "sql"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "integration_version", "1.1.0"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.input_id", "sql-sql/metrics"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.enabled", "true"),
resource.TestCheckNoResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.vars_json"),
resource.TestCheckResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", `{"sql.sql":{"enabled":true,"vars":{"data_stream.dataset":"sql","driver":"mysql","hosts":["root:test@tcp(127.0.0.1:3306)/"],"merge_results":false,"period":"1m","processors":"","sql_queries":"- query: SHOW GLOBAL STATUS LIKE 'Innodb_system%'\n response_format: variables\n \n","ssl":""}}}`),
),
},
{
SkipFunc: func() (bool, error) {
return versionutils.CheckIfVersionIsUnsupported(minVersionSqlIntegration)()
},
ResourceName: "elasticstack_fleet_integration_policy.test_policy",
Config: testAccResourceIntegrationPolicySecretsIds(policyName, "created"),
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{"input.0.streams_json"},
Check: resource.ComposeTestCheckFunc(
resource.TestMatchResourceAttr("elasticstack_fleet_integration_policy.test_policy", "input.0.streams_json", regexp.MustCompile(`"hosts":{"ids":["\S+"],"isSecretRef":true}`)),
),
},
},
},
})
})
}

Expand Down Expand Up @@ -451,3 +493,39 @@ resource "elasticstack_fleet_integration_policy" "test_policy" {
}
`, common, id, key, id)
}

func testAccResourceIntegrationPolicySecretsIds(id string, key string) string {
common := testAccResourceIntegrationPolicyCommon(id, "sql", "1.1.0")
return fmt.Sprintf(`
%s

resource "elasticstack_fleet_integration_policy" "test_policy" {
name = "%s"
namespace = "default"
description = "SQL Integration Policy"
agent_policy_id = elasticstack_fleet_agent_policy.test_policy.policy_id
integration_name = elasticstack_fleet_integration.test_policy.name
integration_version = elasticstack_fleet_integration.test_policy.version

input {
input_id = "sql-sql/metrics"
enabled = true
streams_json = jsonencode({
"sql.sql" : {
"enabled" : true,
"vars" : {
"hosts" : ["root:test@tcp(127.0.0.1:3306)/"],
"period" : "1m",
"driver" : "mysql",
"sql_queries" : "- query: SHOW GLOBAL STATUS LIKE 'Innodb_system%%'\n response_format: variables\n \n",
"merge_results" : false,
"ssl" : "",
"data_stream.dataset" : "sql",
"processors" : ""
}
}
})
}
}
`, common, id)
}
36 changes: 31 additions & 5 deletions internal/fleet/integration_policy/secrets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,21 @@ func HandleRespSecrets(ctx context.Context, resp *kbapi.PackagePolicy, private p
}

handleVar := func(key string, mval map[string]any, vars map[string]any) {
refID := mval["id"].(string)
if original, ok := secrets[refID]; ok {
vars[key] = original
if refID, ok := mval["id"]; ok {
if original, ok := secrets[refID.(string)]; ok {
vars[key] = original
}
} else if ids, ok := mval["ids"]; ok {
values := []any{}
for _, id := range ids.([]any) {
if original, ok := secrets[id.(string)]; ok {
values = append(values, original)
}
}

if len(values) > 0 {
vars[key] = values
}
}
}

Expand Down Expand Up @@ -136,8 +148,22 @@ func HandleReqRespSecrets(ctx context.Context, req kbapi.PackagePolicyRequest, r
}
}

refID := mval["id"].(string)
secrets[refID] = original
if refID, ok := mval["id"]; ok {
secrets[refID.(string)] = original
} else if ids, ok := mval["ids"]; ok {
originals, ok := original.([]any)
if !ok || len(originals) != len(ids.([]any)) {
diags.AddError("mismatched secret ref ids and original values", "the number of secret ref ids does not match the number of original values")
return
}

// Map each id to the corresponding original value by position.
// The API does not return the original value with the id,
// so we have to assume the order is preserved.
for i, id := range ids.([]any) {
secrets[id.(string)] = originals[i]
}
}
}
}

Expand Down
33 changes: 25 additions & 8 deletions internal/fleet/integration_policy/secrets_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,12 @@ func TestHandleRespSecrets(t *testing.T) {
t.Parallel()

ctx := context.Background()
private := privateData{"secrets": `{"known-secret":"secret"}`}
private := privateData{"secrets": `{"known-secret":"secret", "known-secret-1":"secret1", "known-secret-2":"secret2"}`}

secretRefs := &[]kbapi.PackagePolicySecretRef{
{Id: "known-secret"},
{Id: "known-secret-1"},
{Id: "known-secret-2"},
}

tests := []struct {
Expand Down Expand Up @@ -64,6 +66,11 @@ func TestHandleRespSecrets(t *testing.T) {
input: Map{"k": Map{"isSecretRef": true, "id": "known-secret"}},
want: Map{"k": "secret"},
},
{
name: "converts secret with multiple values",
input: Map{"k": Map{"isSecretRef": true, "ids": []any{"known-secret-1", "known-secret-2"}}},
want: Map{"k": []any{"secret1", "secret2"}},
},
{
name: "converts wrapped secret",
input: Map{"k": Map{"type": "password", "value": Map{"isSecretRef": true, "id": "known-secret"}}},
Expand Down Expand Up @@ -121,7 +128,7 @@ func TestHandleRespSecrets(t *testing.T) {
require.Equal(t, want, got)

// privateData
privateWants := privateData{"secrets": `{"known-secret":"secret"}`}
privateWants := privateData{"secrets": `{"known-secret":"secret","known-secret-1":"secret1","known-secret-2":"secret2"}`}
require.Equal(t, privateWants, private)
})
}
Expand All @@ -134,6 +141,8 @@ func TestHandleReqRespSecrets(t *testing.T) {

secretRefs := &[]kbapi.PackagePolicySecretRef{
{Id: "known-secret"},
{Id: "known-secret-1"},
{Id: "known-secret-2"},
}

tests := []struct {
Expand Down Expand Up @@ -166,6 +175,12 @@ func TestHandleReqRespSecrets(t *testing.T) {
respInput: Map{"k": Map{"isSecretRef": true, "id": "known-secret"}},
want: Map{"k": "secret"},
},
{
name: "converts secret with multiple values",
reqInput: Map{"k": []any{"secret1", "secret2"}},
respInput: Map{"k": Map{"isSecretRef": true, "ids": []any{"known-secret-1", "known-secret-2"}}},
want: Map{"k": []any{"secret1", "secret2"}},
},
{
name: "converts wrapped secret",
reqInput: Map{"k": "secret"},
Expand Down Expand Up @@ -236,13 +251,15 @@ func TestHandleReqRespSecrets(t *testing.T) {
want = *(*wants.Inputs["input1"].Streams)["stream1"].Vars
require.Equal(t, want, got)

if v, ok := (*req.Vars)["k"]; ok && v == "secret" {
privateWants := privateData{"secrets": `{"known-secret":"secret"}`}
require.Equal(t, privateWants, private)
} else {
privateWants := privateData{"secrets": `{}`}
require.Equal(t, privateWants, private)
privateWants := privateData{"secrets": `{}`}
if v, ok := (*req.Vars)["k"]; ok {
if s, ok := v.(string); ok && s == "secret" {
privateWants = privateData{"secrets": `{"known-secret":"secret"}`}
} else if _, ok := v.([]any); ok {
privateWants = privateData{"secrets": `{"known-secret-1":"secret1","known-secret-2":"secret2"}`}
}
}
require.Equal(t, privateWants, private)
})
}
}
Loading