Skip to content

Commit add9bdd

Browse files
tobiotobio
authored
Add support for Kafka Fleet output types. (elastic#1302)
* Update bundled Kibana client generation * Add support for Kafka fleet output * Docs * Changelog * Extract common validation from the conditional validator * Remove failureMessage from the conditional validator * Use better schema types, and split up model code Single entry attributes should be objects, not lists * Update docs from schema change * Split the models file up even more
1 parent e29b951 commit add9bdd

File tree

22 files changed

+3291
-531
lines changed

22 files changed

+3291
-531
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
- Migrate `elasticstack_kibana_action_connector` to the Terraform plugin framework ([#1269](https://github.com/elastic/terraform-provider-elasticstack/pull/1269))
1414
- Migrate `elasticstack_elasticsearch_security_role_mapping` resource and data source to Terraform Plugin Framework ([#1279](https://github.com/elastic/terraform-provider-elasticstack/pull/1279))
1515
- Add support for `inactivity_timeout` in `elasticstack_fleet_agent_policy` ([#641](https://github.com/elastic/terraform-provider-elasticstack/issues/641))
16+
- Add support for `kafka` output types in `elasticstack_fleet_output` ([#1302](https://github.com/elastic/terraform-provider-elasticstack/pull/1302))
1617
- Add support for `prevent_initial_backfill` to `elasticstack_kibana_slo` ([#1071](https://github.com/elastic/terraform-provider-elasticstack/pull/1071))
1718
- [Refactor] Regenerate the SLO client using the current OpenAPI spec ([#1303](https://github.com/elastic/terraform-provider-elasticstack/pull/1303))
1819
- Add support for `data_view_id` in the `elasticstack_kibana_slo` resource ([#1305](https://github.com/elastic/terraform-provider-elasticstack/pull/1305))

‎docs/resources/fleet_output.md‎

Lines changed: 235 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
21
---
32
# generated by https://github.com/hashicorp/terraform-plugin-docs
43
page_title: "elasticstack_fleet_output Resource - terraform-provider-elasticstack"
@@ -13,6 +12,8 @@ Creates a new Fleet Output.
1312

1413
## Example Usage
1514

15+
### Basic output
16+
1617
```terraform
1718
provider "elasticstack" {
1819
kibana {}
@@ -32,6 +33,168 @@ resource "elasticstack_fleet_output" "test_output" {
3233
}
3334
```
3435

36+
### Basic Kafka output
37+
38+
```terraform
39+
terraform {
40+
required_providers {
41+
elasticstack = {
42+
source = "elastic/elasticstack"
43+
version = "~> 0.11"
44+
}
45+
}
46+
}
47+
48+
provider "elasticstack" {
49+
elasticsearch {}
50+
kibana {}
51+
}
52+
53+
# Basic Kafka Fleet Output
54+
resource "elasticstack_fleet_output" "kafka_basic" {
55+
name = "Basic Kafka Output"
56+
output_id = "kafka-basic-output"
57+
type = "kafka"
58+
default_integrations = false
59+
default_monitoring = false
60+
61+
hosts = [
62+
"kafka:9092"
63+
]
64+
65+
# Basic Kafka configuration
66+
kafka = {
67+
auth_type = "user_pass"
68+
username = "kafka_user"
69+
password = "kafka_password"
70+
topic = "elastic-beats"
71+
partition = "hash"
72+
compression = "gzip"
73+
required_acks = 1
74+
75+
headers = [
76+
{
77+
key = "environment"
78+
value = "production"
79+
}
80+
]
81+
}
82+
}
83+
```
84+
85+
### Advanced Kafka output
86+
87+
```terraform
88+
terraform {
89+
required_providers {
90+
elasticstack = {
91+
source = "elastic/elasticstack"
92+
version = "~> 0.11"
93+
}
94+
}
95+
}
96+
97+
provider "elasticstack" {
98+
elasticsearch {}
99+
kibana {}
100+
}
101+
102+
# Advanced Kafka Fleet Output with SSL authentication
103+
resource "elasticstack_fleet_output" "kafka_advanced" {
104+
name = "Advanced Kafka Output"
105+
output_id = "kafka-advanced-output"
106+
type = "kafka"
107+
default_integrations = false
108+
default_monitoring = false
109+
110+
hosts = [
111+
"kafka1:9092",
112+
"kafka2:9092",
113+
"kafka3:9092"
114+
]
115+
116+
# Advanced Kafka configuration
117+
kafka = {
118+
auth_type = "ssl"
119+
topic = "elastic-logs"
120+
partition = "round_robin"
121+
compression = "snappy"
122+
required_acks = -1
123+
broker_timeout = 10
124+
timeout = 30
125+
version = "2.6.0"
126+
client_id = "elastic-beats-client"
127+
128+
# Custom headers for message metadata
129+
headers = [
130+
{
131+
key = "datacenter"
132+
value = "us-west-1"
133+
},
134+
{
135+
key = "service"
136+
value = "beats"
137+
},
138+
{
139+
key = "environment"
140+
value = "production"
141+
}
142+
]
143+
144+
# Hash-based partitioning
145+
hash = {
146+
hash = "host.name"
147+
random = false
148+
}
149+
150+
# SASL configuration
151+
sasl = {
152+
mechanism = "SCRAM-SHA-256"
153+
}
154+
}
155+
156+
# SSL configuration (reusing common SSL block)
157+
ssl = {
158+
certificate_authorities = [
159+
file("${path.module}/ca.crt")
160+
]
161+
certificate = file("${path.module}/client.crt")
162+
key = file("${path.module}/client.key")
163+
}
164+
165+
# Additional YAML configuration for advanced settings
166+
config_yaml = yamlencode({
167+
"ssl.verification_mode" = "full"
168+
"ssl.supported_protocols" = ["TLSv1.2", "TLSv1.3"]
169+
"max.message.bytes" = 1000000
170+
})
171+
}
172+
173+
# Example showing round-robin partitioning with event grouping
174+
resource "elasticstack_fleet_output" "kafka_round_robin" {
175+
name = "Kafka Round Robin Output"
176+
output_id = "kafka-round-robin-output"
177+
type = "kafka"
178+
default_integrations = false
179+
default_monitoring = false
180+
181+
hosts = ["kafka:9092"]
182+
183+
kafka = {
184+
auth_type = "none"
185+
topic = "elastic-metrics"
186+
partition = "round_robin"
187+
compression = "lz4"
188+
189+
round_robin = [
190+
{
191+
group_events = 100
192+
}
193+
]
194+
}
195+
}
196+
```
197+
35198
<!-- schema generated by tfplugindocs -->
36199
## Schema
37200

@@ -48,14 +211,83 @@ resource "elasticstack_fleet_output" "test_output" {
48211
- `default_integrations` (Boolean) Make this output the default for agent integrations.
49212
- `default_monitoring` (Boolean) Make this output the default for agent monitoring.
50213
- `hosts` (List of String) A list of hosts.
214+
- `kafka` (Attributes) Kafka-specific configuration. (see [below for nested schema](#nestedatt--kafka))
51215
- `output_id` (String) Unique identifier of the output.
52-
- `ssl` (Block List) SSL configuration. (see [below for nested schema](#nestedblock--ssl))
216+
- `ssl` (Attributes) SSL configuration. (see [below for nested schema](#nestedatt--ssl))
53217

54218
### Read-Only
55219

56220
- `id` (String) The ID of this resource.
57221

58-
<a id="nestedblock--ssl"></a>
222+
<a id="nestedatt--kafka"></a>
223+
### Nested Schema for `kafka`
224+
225+
Optional:
226+
227+
- `auth_type` (String) Authentication type for Kafka output.
228+
- `broker_timeout` (Number) Kafka broker timeout.
229+
- `client_id` (String) Kafka client ID.
230+
- `compression` (String) Compression type for Kafka output.
231+
- `compression_level` (Number) Compression level for Kafka output.
232+
- `connection_type` (String) Connection type for Kafka output.
233+
- `hash` (Attributes) Hash configuration for Kafka partition. (see [below for nested schema](#nestedatt--kafka--hash))
234+
- `headers` (Attributes List) Headers for Kafka messages. (see [below for nested schema](#nestedatt--kafka--headers))
235+
- `key` (String) Key field for Kafka messages.
236+
- `partition` (String) Partition strategy for Kafka output.
237+
- `password` (String, Sensitive) Password for Kafka authentication.
238+
- `random` (Attributes) Random configuration for Kafka partition. (see [below for nested schema](#nestedatt--kafka--random))
239+
- `required_acks` (Number) Number of acknowledgments required for Kafka output.
240+
- `round_robin` (Attributes) Round robin configuration for Kafka partition. (see [below for nested schema](#nestedatt--kafka--round_robin))
241+
- `sasl` (Attributes) SASL configuration for Kafka authentication. (see [below for nested schema](#nestedatt--kafka--sasl))
242+
- `timeout` (Number) Timeout for Kafka output.
243+
- `topic` (String) Kafka topic.
244+
- `username` (String) Username for Kafka authentication.
245+
- `version` (String) Kafka version.
246+
247+
<a id="nestedatt--kafka--hash"></a>
248+
### Nested Schema for `kafka.hash`
249+
250+
Optional:
251+
252+
- `hash` (String) Hash field.
253+
- `random` (Boolean) Use random hash.
254+
255+
256+
<a id="nestedatt--kafka--headers"></a>
257+
### Nested Schema for `kafka.headers`
258+
259+
Required:
260+
261+
- `key` (String) Header key.
262+
- `value` (String) Header value.
263+
264+
265+
<a id="nestedatt--kafka--random"></a>
266+
### Nested Schema for `kafka.random`
267+
268+
Optional:
269+
270+
- `group_events` (Number) Number of events to group.
271+
272+
273+
<a id="nestedatt--kafka--round_robin"></a>
274+
### Nested Schema for `kafka.round_robin`
275+
276+
Optional:
277+
278+
- `group_events` (Number) Number of events to group.
279+
280+
281+
<a id="nestedatt--kafka--sasl"></a>
282+
### Nested Schema for `kafka.sasl`
283+
284+
Optional:
285+
286+
- `mechanism` (String) SASL mechanism.
287+
288+
289+
290+
<a id="nestedatt--ssl"></a>
59291
### Nested Schema for `ssl`
60292

61293
Required:

‎examples/resources/elasticstack_fleet_output/kafka_advanced.tf‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
terraform {
2+
required_providers {
3+
elasticstack = {
4+
source = "elastic/elasticstack"
5+
version = "~> 0.11"
6+
}
7+
}
8+
}
9+
10+
provider "elasticstack" {
11+
elasticsearch {}
12+
kibana {}
13+
}
14+
15+
# Advanced Kafka Fleet Output with SSL authentication
16+
resource "elasticstack_fleet_output" "kafka_advanced" {
17+
name = "Advanced Kafka Output"
18+
output_id = "kafka-advanced-output"
19+
type = "kafka"
20+
default_integrations = false
21+
default_monitoring = false
22+
23+
hosts = [
24+
"kafka1:9092",
25+
"kafka2:9092",
26+
"kafka3:9092"
27+
]
28+
29+
# Advanced Kafka configuration
30+
kafka = {
31+
auth_type = "ssl"
32+
topic = "elastic-logs"
33+
partition = "round_robin"
34+
compression = "snappy"
35+
required_acks = -1
36+
broker_timeout = 10
37+
timeout = 30
38+
version = "2.6.0"
39+
client_id = "elastic-beats-client"
40+
41+
# Custom headers for message metadata
42+
headers = [
43+
{
44+
key = "datacenter"
45+
value = "us-west-1"
46+
},
47+
{
48+
key = "service"
49+
value = "beats"
50+
},
51+
{
52+
key = "environment"
53+
value = "production"
54+
}
55+
]
56+
57+
# Hash-based partitioning
58+
hash = {
59+
hash = "host.name"
60+
random = false
61+
}
62+
63+
# SASL configuration
64+
sasl = {
65+
mechanism = "SCRAM-SHA-256"
66+
}
67+
}
68+
69+
# SSL configuration (reusing common SSL block)
70+
ssl = {
71+
certificate_authorities = [
72+
file("${path.module}/ca.crt")
73+
]
74+
certificate = file("${path.module}/client.crt")
75+
key = file("${path.module}/client.key")
76+
}
77+
78+
# Additional YAML configuration for advanced settings
79+
config_yaml = yamlencode({
80+
"ssl.verification_mode" = "full"
81+
"ssl.supported_protocols" = ["TLSv1.2", "TLSv1.3"]
82+
"max.message.bytes" = 1000000
83+
})
84+
}
85+
86+
# Example showing round-robin partitioning with event grouping
87+
resource "elasticstack_fleet_output" "kafka_round_robin" {
88+
name = "Kafka Round Robin Output"
89+
output_id = "kafka-round-robin-output"
90+
type = "kafka"
91+
default_integrations = false
92+
default_monitoring = false
93+
94+
hosts = ["kafka:9092"]
95+
96+
kafka = {
97+
auth_type = "none"
98+
topic = "elastic-metrics"
99+
partition = "round_robin"
100+
compression = "lz4"
101+
102+
round_robin = [
103+
{
104+
group_events = 100
105+
}
106+
]
107+
}
108+
}

0 commit comments

Comments
 (0)