Skip to content

Commit 71179fb

Browse files
vbotbuildovichJakeSCahillmfernestclaudegithub-actions[bot]
authored
auto-docs: Update RPCN connector docs (#385)
Co-authored-by: JakeSCahill <45230295+JakeSCahill@users.noreply.github.com> Co-authored-by: Michael Ernest <michael.ernest@gmail.com> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com> Co-authored-by: Joyce Fee <joyce@redpanda.com> Co-authored-by: micheleRP <michele@redpanda.com>
1 parent 03e1d53 commit 71179fb

File tree

15 files changed

+1002
-74
lines changed

15 files changed

+1002
-74
lines changed

‎docs-data/connect-4.81.0.json‎

Lines changed: 296 additions & 21 deletions
Large diffs are not rendered by default.

‎docs-data/overrides.json‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1783,7 +1783,7 @@
17831783
},
17841784
{
17851785
"name": "seed_brokers",
1786-
"$ref": "#/definitions/seed_brokers"
1786+
"description": "A list of broker addresses to connect to in order. Use commas to separate multiple addresses in a single list item. Optional when `seed_brokers` is configured in a top-level `redpanda` block."
17871787
},
17881788
{
17891789
"name": "tls",
@@ -4004,6 +4004,22 @@
40044004
]
40054005
}
40064006
},
4007+
{
4008+
"name": "iceberg",
4009+
"config": {
4010+
"children": [
4011+
{
4012+
"name": "catalog",
4013+
"children": [
4014+
{
4015+
"name": "headers",
4016+
"type": "object"
4017+
}
4018+
]
4019+
}
4020+
]
4021+
}
4022+
},
40074023
{
40084024
"name": "kafka",
40094025
"config": {
@@ -4860,7 +4876,7 @@
48604876
"children": [
48614877
{
48624878
"name": "seed_brokers",
4863-
"$ref": "#/definitions/seed_brokers"
4879+
"description": "A list of broker addresses to connect to in order. Use commas to separate multiple addresses in a single list item. Optional when `seed_brokers` is configured in a top-level `redpanda` block."
48644880
},
48654881
{
48664882
"name": "tls",

‎modules/ROOT/nav.adoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
** xref:components:inputs/about.adoc[]
4747
*** xref:components:inputs/amqp_0_9.adoc[]
4848
*** xref:components:inputs/amqp_1.adoc[]
49+
*** xref:components:inputs/aws_cloudwatch_logs.adoc[]
4950
*** xref:components:inputs/aws_dynamodb_cdc.adoc[]
5051
*** xref:components:inputs/aws_kinesis.adoc[]
5152
*** xref:components:inputs/aws_s3.adoc[]

‎modules/components/attachments/connect-4.81.0.json‎

Lines changed: 279 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18770,7 +18770,7 @@
1877018770
"name": "seed_brokers",
1877118771
"type": "string",
1877218772
"kind": "array",
18773-
"description": "A list of broker addresses to connect to in order. Use commas to separate multiple addresses in a single list item.",
18773+
"description": "A list of broker addresses to connect to in order. Use commas to separate multiple addresses in a single list item. Optional when `seed_brokers` is configured in a top-level `redpanda` block.",
1877418774
"is_optional": true,
1877518775
"examples": [
1877618776
[
@@ -23744,6 +23744,266 @@
2374423744
"cloudSupported": false,
2374523745
"requiresCgo": false
2374623746
},
23747+
{
23748+
"name": "aws_cloudwatch_logs",
23749+
"type": "input",
23750+
"status": "stable",
23751+
"plugin": true,
23752+
"summary": "Consumes log events from AWS CloudWatch Logs.",
23753+
"description": "\nPolls CloudWatch Log Groups for log events. Supports filtering by log streams, CloudWatch filter patterns, and configurable start times.\n\nEach log event becomes a separate message with metadata including the log group name, log stream name, timestamp, and ingestion time.\n\nIMPORTANT: This input tracks its position in memory only. If the process restarts, it will resume from the configured start_time (or the beginning if not set). For exactly-once processing, you should configure an appropriate start_time or implement idempotent downstream processing.\n\n## Credentials\n\nBy default Redpanda Connect will use a shared credentials file when connecting to AWS services. It's also possible to set them explicitly at the component level, allowing you to transfer data across accounts. You can find out more in xref:guides:cloud/aws.adoc[].\n\n## Metadata\n\nThis input adds the following metadata fields to each message:\n\n- `cloudwatch_log_group` - The name of the log group\n- `cloudwatch_log_stream` - The name of the log stream\n- `cloudwatch_timestamp` - The timestamp of the log event (Unix milliseconds)\n- `cloudwatch_ingestion_time` - The ingestion timestamp (Unix milliseconds)\n- `cloudwatch_event_id` - The unique event ID\n\nYou can access these metadata fields using xref:guides:bloblang/about.adoc[Bloblang].\n",
23754+
"categories": [
23755+
"Services",
23756+
"AWS"
23757+
],
23758+
"config": {
23759+
"name": "",
23760+
"type": "object",
23761+
"kind": "scalar",
23762+
"children": [
23763+
{
23764+
"name": "log_group_name",
23765+
"type": "string",
23766+
"kind": "scalar",
23767+
"description": "The name of the CloudWatch Log Group to consume from.",
23768+
"examples": [
23769+
"my-app-logs"
23770+
]
23771+
},
23772+
{
23773+
"name": "log_stream_names",
23774+
"type": "string",
23775+
"kind": "array",
23776+
"description": "An optional list of log stream names to consume from. If not set, events from all streams in the log group will be consumed.",
23777+
"is_optional": true,
23778+
"examples": [
23779+
[
23780+
"stream-1",
23781+
"stream-2"
23782+
]
23783+
]
23784+
},
23785+
{
23786+
"name": "log_stream_prefix",
23787+
"type": "string",
23788+
"kind": "scalar",
23789+
"description": "An optional log stream name prefix to filter streams. Only streams starting with this prefix will be consumed.",
23790+
"is_optional": true,
23791+
"examples": [
23792+
"prod-"
23793+
]
23794+
},
23795+
{
23796+
"name": "filter_pattern",
23797+
"type": "string",
23798+
"kind": "scalar",
23799+
"description": "An optional CloudWatch Logs filter pattern to apply when querying log events. See AWS documentation for filter pattern syntax.",
23800+
"is_optional": true,
23801+
"examples": [
23802+
"[ERROR]"
23803+
]
23804+
},
23805+
{
23806+
"name": "start_time",
23807+
"type": "string",
23808+
"kind": "scalar",
23809+
"description": "The time to start consuming log events from. Can be an RFC3339 timestamp (e.g., `2024-01-01T00:00:00Z`) or the string `now` to start consuming from the current time. If not set, starts from the beginning of available logs.",
23810+
"is_optional": true,
23811+
"examples": [
23812+
"2024-01-01T00:00:00Z",
23813+
"now"
23814+
]
23815+
},
23816+
{
23817+
"name": "poll_interval",
23818+
"type": "string",
23819+
"kind": "scalar",
23820+
"description": "The interval at which to poll for new log events.",
23821+
"default": "5s"
23822+
},
23823+
{
23824+
"name": "limit",
23825+
"type": "int",
23826+
"kind": "scalar",
23827+
"description": "The maximum number of log events to return in a single API call. Valid range: 1-10000.",
23828+
"is_advanced": true,
23829+
"default": 1000,
23830+
"linter": "root = if this < 1 || this > 10000 { [\"limit must be between 1 and 10000\"] }"
23831+
},
23832+
{
23833+
"name": "structured_log",
23834+
"type": "bool",
23835+
"kind": "scalar",
23836+
"description": "Whether to output log events as structured JSON objects with all metadata fields, or as plain text messages with metadata in message metadata.",
23837+
"is_advanced": true,
23838+
"default": true
23839+
},
23840+
{
23841+
"name": "api_timeout",
23842+
"type": "string",
23843+
"kind": "scalar",
23844+
"description": "The maximum time to wait for an API request to complete.",
23845+
"is_advanced": true,
23846+
"default": "30s"
23847+
},
23848+
{
23849+
"name": "auto_replay_nacks",
23850+
"type": "bool",
23851+
"kind": "scalar",
23852+
"description": "Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation.",
23853+
"default": true
23854+
},
23855+
{
23856+
"name": "region",
23857+
"type": "string",
23858+
"kind": "scalar",
23859+
"description": "The AWS region to target.",
23860+
"is_advanced": true,
23861+
"is_optional": true
23862+
},
23863+
{
23864+
"name": "endpoint",
23865+
"type": "string",
23866+
"kind": "scalar",
23867+
"description": "Allows you to specify a custom endpoint for the AWS API.",
23868+
"is_advanced": true,
23869+
"is_optional": true
23870+
},
23871+
{
23872+
"name": "tcp",
23873+
"type": "object",
23874+
"kind": "scalar",
23875+
"description": "TCP socket configuration.",
23876+
"is_advanced": true,
23877+
"is_optional": true,
23878+
"children": [
23879+
{
23880+
"name": "connect_timeout",
23881+
"type": "string",
23882+
"kind": "scalar",
23883+
"description": "Maximum amount of time a dial will wait for a connect to complete. Zero disables.",
23884+
"is_advanced": true,
23885+
"default": "0s"
23886+
},
23887+
{
23888+
"name": "keep_alive",
23889+
"type": "object",
23890+
"kind": "scalar",
23891+
"description": "TCP keep-alive probe configuration.",
23892+
"is_advanced": true,
23893+
"is_optional": true,
23894+
"children": [
23895+
{
23896+
"name": "idle",
23897+
"type": "string",
23898+
"kind": "scalar",
23899+
"description": "Duration the connection must be idle before sending the first keep-alive probe. Zero defaults to 15s. Negative values disable keep-alive probes.",
23900+
"is_advanced": true,
23901+
"default": "15s"
23902+
},
23903+
{
23904+
"name": "interval",
23905+
"type": "string",
23906+
"kind": "scalar",
23907+
"description": "Duration between keep-alive probes. Zero defaults to 15s.",
23908+
"is_advanced": true,
23909+
"default": "15s"
23910+
},
23911+
{
23912+
"name": "count",
23913+
"type": "int",
23914+
"kind": "scalar",
23915+
"description": "Maximum unanswered keep-alive probes before dropping the connection. Zero defaults to 9.",
23916+
"is_advanced": true,
23917+
"default": 9
23918+
}
23919+
]
23920+
},
23921+
{
23922+
"name": "tcp_user_timeout",
23923+
"type": "string",
23924+
"kind": "scalar",
23925+
"description": "Maximum time to wait for acknowledgment of transmitted data before killing the connection. Linux-only (kernel 2.6.37+), ignored on other platforms. When enabled, keep_alive.idle must be greater than this value per RFC 5482. Zero disables.",
23926+
"is_advanced": true,
23927+
"default": "0s"
23928+
}
23929+
]
23930+
},
23931+
{
23932+
"name": "credentials",
23933+
"type": "object",
23934+
"kind": "scalar",
23935+
"description": "Optional manual configuration of AWS credentials to use. More information can be found in xref:guides:cloud/aws.adoc[].",
23936+
"is_advanced": true,
23937+
"is_optional": true,
23938+
"children": [
23939+
{
23940+
"name": "profile",
23941+
"type": "string",
23942+
"kind": "scalar",
23943+
"description": "A profile from `~/.aws/credentials` to use.",
23944+
"is_advanced": true,
23945+
"is_optional": true
23946+
},
23947+
{
23948+
"name": "id",
23949+
"type": "string",
23950+
"kind": "scalar",
23951+
"description": "The ID of credentials to use.",
23952+
"is_advanced": true,
23953+
"is_optional": true
23954+
},
23955+
{
23956+
"name": "secret",
23957+
"type": "string",
23958+
"kind": "scalar",
23959+
"description": "The secret for the credentials being used.",
23960+
"is_advanced": true,
23961+
"is_optional": true,
23962+
"is_secret": true,
23963+
"scrubber": "root = if this != null && this != \"\" && !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"
23964+
},
23965+
{
23966+
"name": "token",
23967+
"type": "string",
23968+
"kind": "scalar",
23969+
"description": "The token for the credentials being used, required when using short term credentials.",
23970+
"is_advanced": true,
23971+
"is_optional": true
23972+
},
23973+
{
23974+
"name": "from_ec2_role",
23975+
"type": "bool",
23976+
"kind": "scalar",
23977+
"description": "Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].",
23978+
"is_advanced": true,
23979+
"is_optional": true,
23980+
"version": "4.2.0"
23981+
},
23982+
{
23983+
"name": "role",
23984+
"type": "string",
23985+
"kind": "scalar",
23986+
"description": "A role ARN to assume.",
23987+
"is_advanced": true,
23988+
"is_optional": true
23989+
},
23990+
{
23991+
"name": "role_external_id",
23992+
"type": "string",
23993+
"kind": "scalar",
23994+
"description": "An external ID to provide when assuming a role.",
23995+
"is_advanced": true,
23996+
"is_optional": true
23997+
}
23998+
]
23999+
}
24000+
],
24001+
"linter": "\nroot = if this.log_stream_names.or([]).length() > 0 && this.exists(\"log_stream_prefix\") {\n \"cannot specify both log_stream_names and log_stream_prefix\"\n}\n"
24002+
},
24003+
"version": "4.81.0",
24004+
"requiresCgo": true,
24005+
"cloudSupported": false
24006+
},
2374724007
{
2374824008
"name": "tigerbeetle_cdc",
2374924009
"type": "input",
@@ -41042,7 +41302,7 @@
4104241302
"name": "seed_brokers",
4104341303
"type": "string",
4104441304
"kind": "array",
41045-
"description": "A list of broker addresses to connect to in order. Use commas to separate multiple addresses in a single list item.",
41305+
"description": "A list of broker addresses to connect to in order. Use commas to separate multiple addresses in a single list item. Optional when `seed_brokers` is configured in a top-level `redpanda` block.",
4104641306
"is_optional": true,
4104741307
"examples": [
4104841308
[
@@ -46811,6 +47071,22 @@
4681147071
"is_advanced": true,
4681247072
"default": false
4681347073
},
47074+
{
47075+
"name": "region",
47076+
"type": "string",
47077+
"kind": "scalar",
47078+
"description": "The AWS region to target.",
47079+
"is_advanced": true,
47080+
"is_optional": true
47081+
},
47082+
{
47083+
"name": "endpoint",
47084+
"type": "string",
47085+
"kind": "scalar",
47086+
"description": "Allows you to specify a custom endpoint for the AWS API.",
47087+
"is_advanced": true,
47088+
"is_optional": true
47089+
},
4681447090
{
4681547091
"name": "tcp",
4681647092
"type": "object",
@@ -46920,8 +47196,7 @@
4692047196
"description": "Use the credentials of a host EC2 machine configured to assume https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html[an IAM role associated with the instance^].",
4692147197
"is_advanced": true,
4692247198
"is_optional": true,
46923-
"version": "4.2.0",
46924-
"default": false
47199+
"version": "4.2.0"
4692547200
},
4692647201
{
4692747202
"name": "role",

‎modules/components/examples/advanced/inputs/aws_cloudwatch_logs.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
inputs:
2+
label: ""
3+
aws_cloudwatch_logs:
4+
log_group_name: "" # No default (required)
5+
log_stream_names: [] # No default (optional)
6+
log_stream_prefix: "" # No default (optional)
7+
filter_pattern: "" # No default (optional)
8+
start_time: "" # No default (optional)
9+
poll_interval: 5s
10+
limit: 1000
11+
structured_log: true
12+
api_timeout: 30s
13+
auto_replay_nacks: true
14+
region: "" # No default (optional)
15+
endpoint: "" # No default (optional)
16+
tcp:
17+
connect_timeout: 0s
18+
keep_alive:
19+
idle: 15s
20+
interval: 15s
21+
count: 9
22+
tcp_user_timeout: 0s
23+
credentials:
24+
profile: "" # No default (optional)
25+
id: "" # No default (optional)
26+
secret: "" # No default (optional)
27+
token: "" # No default (optional)
28+
from_ec2_role: false
29+
role: "" # No default (optional)
30+
role_external_id: "" # No default (optional)

0 commit comments

Comments
 (0)