Skip to content

Commit 6f705e3

Browse files
vbotbuildovichJakeSCahill
vbotbuildovich
and
JakeSCahill
authored
auto-docs: Update RPCN connector docs (#340)
Co-authored-by: JakeSCahill <[email protected]> Co-authored-by: JakeSCahill <[email protected]>
1 parent e307230 commit 6f705e3

File tree

10 files changed

+1599
-81
lines changed

10 files changed

+1599
-81
lines changed

‎antora.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ asciidoc:
1212
project-github: redpanda-data/connect
1313
full-version: 24.1.4
1414
latest-release-commit: b5ade3f40
15-
latest-connect-version: 4.71.0
15+
latest-connect-version: 4.72.0
1616
page-home-intro: >-
1717
Redpanda Connect is a data streaming service for building scalable, high-performance data pipelines that drive
1818
real-time analytics and actionable business insights. Integrate data across systems with hundreds of prebuilt

docs-data/connect-4.70.0.json renamed to docs-data/connect-4.72.0.json

Lines changed: 1290 additions & 72 deletions
Large diffs are not rendered by default.

‎docs-data/connect-diff-4.71.0_to_4.72.0.json‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{
2+
"comparison": {
3+
"oldVersion": "4.71.0",
4+
"newVersion": "4.72.0",
5+
"timestamp": "2025-12-02T14:07:34.116Z"
6+
},
7+
"summary": {
8+
"newComponents": 0,
9+
"removedComponents": 0,
10+
"newFields": 0,
11+
"removedFields": 0,
12+
"deprecatedComponents": 0,
13+
"deprecatedFields": 0,
14+
"changedDefaults": 0
15+
},
16+
"details": {
17+
"newComponents": [],
18+
"removedComponents": [],
19+
"newFields": [],
20+
"removedFields": [],
21+
"deprecatedComponents": [],
22+
"deprecatedFields": [],
23+
"changedDefaults": []
24+
}
25+
}

modules/components/attachments/connect-4.71.0.json renamed to modules/components/attachments/connect-4.72.0.json

Lines changed: 149 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
2-
"version": "4.71.0",
3-
"date": "2025-11-21T11:58:53Z",
2+
"version": "4.72.0",
3+
"date": "2025-11-28T13:07:57Z",
44
"config": [
55
{
66
"name": "http",
@@ -15578,6 +15578,75 @@
1557815578
"kind": "scalar",
1557915579
"description": "The PostgreSQL endpoint hostname (e.g., mydb.abc123.us-east-1.rds.amazonaws.com).",
1558015580
"is_advanced": true
15581+
},
15582+
{
15583+
"name": "id",
15584+
"type": "string",
15585+
"kind": "scalar",
15586+
"description": "The ID of credentials to use.",
15587+
"is_advanced": true,
15588+
"is_optional": true
15589+
},
15590+
{
15591+
"name": "secret",
15592+
"type": "string",
15593+
"kind": "scalar",
15594+
"description": "The secret for the credentials being used.",
15595+
"is_advanced": true,
15596+
"is_optional": true,
15597+
"is_secret": true,
15598+
"scrubber": "root = if this != null && this != \"\" && !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"
15599+
},
15600+
{
15601+
"name": "token",
15602+
"type": "string",
15603+
"kind": "scalar",
15604+
"description": "The token for the credentials being used, required when using short term credentials.",
15605+
"is_advanced": true,
15606+
"is_optional": true
15607+
},
15608+
{
15609+
"name": "role",
15610+
"type": "string",
15611+
"kind": "scalar",
15612+
"description": "Optional AWS IAM role ARN to assume for authentication. Alternatively, use `roles` array for role chaining instead.",
15613+
"is_advanced": true,
15614+
"is_optional": true
15615+
},
15616+
{
15617+
"name": "role_external_id",
15618+
"type": "string",
15619+
"kind": "scalar",
15620+
"description": "Optional external ID for the role assumption. Only used with the `role` field. Alternatively, use `roles` array for role chaining instead.",
15621+
"is_advanced": true,
15622+
"is_optional": true
15623+
},
15624+
{
15625+
"name": "roles",
15626+
"type": "object",
15627+
"kind": "array",
15628+
"description": "Optional array of AWS IAM roles to assume for authentication. Roles can be assumed in sequence, enabling chaining for purposes such as cross-account access. Each role can optionally specify an external ID.",
15629+
"is_advanced": true,
15630+
"is_optional": true,
15631+
"children": [
15632+
{
15633+
"name": "role",
15634+
"type": "string",
15635+
"kind": "scalar",
15636+
"description": "AWS IAM role ARN to assume.",
15637+
"is_advanced": true,
15638+
"default": ""
15639+
},
15640+
{
15641+
"name": "role_external_id",
15642+
"type": "string",
15643+
"kind": "scalar",
15644+
"description": "Optional external ID for the role assumption.",
15645+
"is_advanced": true,
15646+
"is_optional": true,
15647+
"default": ""
15648+
}
15649+
]
1558115650
}
1558215651
]
1558315652
},
@@ -15993,6 +16062,75 @@
1599316062
"kind": "scalar",
1599416063
"description": "The PostgreSQL endpoint hostname (e.g., mydb.abc123.us-east-1.rds.amazonaws.com).",
1599516064
"is_advanced": true
16065+
},
16066+
{
16067+
"name": "id",
16068+
"type": "string",
16069+
"kind": "scalar",
16070+
"description": "The ID of credentials to use.",
16071+
"is_advanced": true,
16072+
"is_optional": true
16073+
},
16074+
{
16075+
"name": "secret",
16076+
"type": "string",
16077+
"kind": "scalar",
16078+
"description": "The secret for the credentials being used.",
16079+
"is_advanced": true,
16080+
"is_optional": true,
16081+
"is_secret": true,
16082+
"scrubber": "root = if this != null && this != \"\" && !this.trim().re_match(\"\"\"^\\${[0-9A-Za-z_.]+(:((\\${[^}]+})|[^}])*)?}$\"\"\") {\n \"!!!SECRET_SCRUBBED!!!\"\n} else if this == null { \"\" }"
16083+
},
16084+
{
16085+
"name": "token",
16086+
"type": "string",
16087+
"kind": "scalar",
16088+
"description": "The token for the credentials being used, required when using short term credentials.",
16089+
"is_advanced": true,
16090+
"is_optional": true
16091+
},
16092+
{
16093+
"name": "role",
16094+
"type": "string",
16095+
"kind": "scalar",
16096+
"description": "Optional AWS IAM role ARN to assume for authentication. Alternatively, use `roles` array for role chaining instead.",
16097+
"is_advanced": true,
16098+
"is_optional": true
16099+
},
16100+
{
16101+
"name": "role_external_id",
16102+
"type": "string",
16103+
"kind": "scalar",
16104+
"description": "Optional external ID for the role assumption. Only used with the `role` field. Alternatively, use `roles` array for role chaining instead.",
16105+
"is_advanced": true,
16106+
"is_optional": true
16107+
},
16108+
{
16109+
"name": "roles",
16110+
"type": "object",
16111+
"kind": "array",
16112+
"description": "Optional array of AWS IAM roles to assume for authentication. Roles can be assumed in sequence, enabling chaining for purposes such as cross-account access. Each role can optionally specify an external ID.",
16113+
"is_advanced": true,
16114+
"is_optional": true,
16115+
"children": [
16116+
{
16117+
"name": "role",
16118+
"type": "string",
16119+
"kind": "scalar",
16120+
"description": "AWS IAM role ARN to assume.",
16121+
"is_advanced": true,
16122+
"default": ""
16123+
},
16124+
{
16125+
"name": "role_external_id",
16126+
"type": "string",
16127+
"kind": "scalar",
16128+
"description": "Optional external ID for the role assumption.",
16129+
"is_advanced": true,
16130+
"is_optional": true,
16131+
"default": ""
16132+
}
16133+
]
1599616134
}
1599716135
]
1599816136
},
@@ -55726,6 +55864,10 @@
5572655864
"PLAIN",
5572755865
"Plain text authentication."
5572855866
],
55867+
[
55868+
"REDPANDA_CLOUD_SERVICE_ACCOUNT",
55869+
"Redpanda Cloud Service Account authentication when running in Redpanda Cloud."
55870+
],
5572955871
[
5573055872
"SCRAM-SHA-256",
5573155873
"SCRAM based authentication as specified in RFC5802."
@@ -55739,7 +55881,7 @@
5573955881
"Disable sasl authentication"
5574055882
]
5574155883
],
55742-
"linter": "\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"
55884+
"linter": "\nlet options = {\n \"aws_msk_iam\": true,\n \"oauthbearer\": true,\n \"plain\": true,\n \"redpanda_cloud_service_account\": true,\n \"scram-sha-256\": true,\n \"scram-sha-512\": true,\n \"none\": true,\n}\n\nroot = if !$options.exists(this.string().lowercase()) {\n {\"type\": 2, \"what\": \"value %v is not a valid option for this field\".format(this.string())}\n}\n"
5574355885
},
5574455886
{
5574555887
"name": "username",
@@ -56177,20 +56319,20 @@
5617756319
}
5617856320
]
5617956321
},
56180-
"description": "\nThe Redpanda tracer exports distributed tracing data to a Redpanda topic, enabling you to monitor and debug your Redpanda Connect pipelines. Traces are exported in OpenTelemetry format as JSON, allowing integration with observability platforms like Jaeger, Grafana Tempo, or custom trace consumers.\n\nThis tracer automatically captures trace spans as messages flow through your pipeline, recording timing information, component metadata, and error details. Use this to:\n\n- **Track message flow** through complex pipelines with multiple processors\n- **Identify performance bottlenecks** by analyzing span durations\n- **Debug failures** by examining trace context and error details\n- **Monitor pipeline health** across distributed Redpanda Connect instances\n- **Correlate activity** across multiple services using trace IDs\n\nThe tracer writes to a dedicated Redpanda topic (default: `otel-traces`) that can be consumed by trace analysis tools. Configure sampling to control trace volume in high-throughput environments.\n\nIMPORTANT: This component is experimental and may change in future releases.\n",
56322+
"description": "\nThe Redpanda tracer exports distributed tracing data to a Redpanda topic, enabling you to monitor and debug your Redpanda Connect pipelines. Traces are exported in OpenTelemetry format as JSON, allowing integration with observability platforms like Jaeger, Grafana Tempo, or custom trace consumers.\n\nThis tracer automatically captures trace spans as messages flow through your pipeline, recording timing information, component metadata, and error details. Use this to:\n\n- **Track message flow** through complex pipelines with multiple processors\n- **Identify performance bottlenecks** by analyzing span durations\n- **Debug failures** by examining trace context and error details\n- **Monitor pipeline health** across distributed Redpanda Connect instances\n- **Correlate activity** across multiple services using trace IDs\n\nThe tracer writes to a dedicated Redpanda topic (default: `otel-traces`) that can be consumed by trace analysis tools. Configure sampling to control trace volume in high-throughput environments.",
5618156323
"examples": [
5618256324
{
56183-
"title": "Basic Tracing Setup",
56325+
"title": "Basic tracing setup",
5618456326
"summary": "Enable tracing for a pipeline, sending all trace data to a local Redpanda cluster.",
5618556327
"config": "\ninput:\n redpanda:\n addresses: [ \"localhost:9092\" ]\n topics: [ \"orders\" ]\n\npipeline:\n processors:\n - mapping: |\n root = this\n root.processed_at = now()\n\noutput:\n redpanda:\n addresses: [ \"localhost:9092\" ]\n topic: \"processed_orders\"\n\ntracer:\n redpanda:\n seed_brokers: [ \"localhost:9092\" ]\n topic: otel-traces\n service: order-processor\n"
5618656328
},
5618756329
{
56188-
"title": "Production Tracing with Sampling",
56330+
"title": "Production tracing with sampling",
5618956331
"summary": "Configure tracing for a production environment with sampling to reduce trace volume and secure authentication.",
5619056332
"config": "\ninput:\n redpanda:\n addresses: [ \"redpanda-prod:9092\" ]\n topics: [ \"events\" ]\n\npipeline:\n processors:\n - branch:\n request_map: 'root = this'\n processors:\n - mapping: 'root.enriched = true'\n\noutput:\n redpanda:\n addresses: [ \"redpanda-prod:9092\" ]\n topic: \"processed_events\"\n\ntracer:\n redpanda:\n seed_brokers: [ \"redpanda-prod:9092\" ]\n topic: otel-traces\n service: event-processor-prod\n sampling:\n enabled: true\n ratio: \"0.1\" # Sample 10% of traces\n tls:\n enabled: true\n sasl:\n mechanism: SCRAM-SHA-512\n username: tracer-user\n password: ${TRACER_PASSWORD}\n"
5619156333
},
5619256334
{
56193-
"title": "Multi-Instance Tracing",
56335+
"title": "Multi-instance tracing",
5619456336
"summary": "Configure tracing for multiple Redpanda Connect instances with unique service names for correlation.",
5619556337
"config": "\n# Instance 1: Ingestion service\ntracer:\n redpanda:\n seed_brokers: [ \"redpanda:9092\" ]\n topic: otel-traces\n service: ingestion-service\n tags:\n environment: production\n region: us-west-2\n instance: ingestion-01\n\n# Instance 2: Processing service\ntracer:\n redpanda:\n seed_brokers: [ \"redpanda:9092\" ]\n topic: otel-traces\n service: processing-service\n tags:\n environment: production\n region: us-west-2\n instance: processing-01\n"
5619656338
}

‎modules/components/examples/advanced/inputs/pg_stream.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,14 @@ inputs:
3131
enabled: false
3232
region: "" # No default (optional)
3333
endpoint: "" # No default (required)
34+
id: "" # No default (optional)
35+
secret: "" # No default (optional)
36+
token: "" # No default (optional)
37+
role: "" # No default (optional)
38+
role_external_id: "" # No default (optional)
39+
roles:
40+
role: ""
41+
role_external_id: ""
3442
auto_replay_nacks: true
3543
batching:
3644
count: 0

‎modules/components/examples/advanced/inputs/postgres_cdc.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,14 @@ inputs:
3131
enabled: false
3232
region: "" # No default (optional)
3333
endpoint: "" # No default (required)
34+
id: "" # No default (optional)
35+
secret: "" # No default (optional)
36+
token: "" # No default (optional)
37+
role: "" # No default (optional)
38+
role_external_id: "" # No default (optional)
39+
roles:
40+
role: ""
41+
role_external_id: ""
3442
auto_replay_nacks: true
3543
batching:
3644
count: 0

‎modules/components/partials/fields/inputs/pg_stream.adoc‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,66 @@ The PostgreSQL endpoint hostname (e.g., mydb.abc123.us-east-1.rds.amazonaws.com)
3434

3535
*Type*: `string`
3636

37+
=== `aws.id`
38+
39+
The ID of credentials to use.
40+
41+
*Type*: `string`
42+
3743
=== `aws.region`
3844

3945
The AWS region where the PostgreSQL instance is located. If no region is specified then the environment default will be used.
4046

4147
*Type*: `string`
4248

49+
=== `aws.role`
50+
51+
Optional AWS IAM role ARN to assume for authentication. Alternatively, use `roles` array for role chaining instead.
52+
53+
*Type*: `string`
54+
55+
=== `aws.role_external_id`
56+
57+
Optional external ID for the role assumption. Only used with the `role` field. Alternatively, use `roles` array for role chaining instead.
58+
59+
*Type*: `string`
60+
61+
=== `aws.roles[]`
62+
63+
Optional array of AWS IAM roles to assume for authentication. Roles can be assumed in sequence, enabling chaining for purposes such as cross-account access. Each role can optionally specify an external ID.
64+
65+
*Type*: `object`
66+
67+
=== `aws.roles[].role`
68+
69+
AWS IAM role ARN to assume.
70+
71+
*Type*: `string`
72+
73+
*Default*: `""`
74+
75+
=== `aws.roles[].role_external_id`
76+
77+
Optional external ID for the role assumption.
78+
79+
*Type*: `string`
80+
81+
*Default*: `""`
82+
83+
=== `aws.secret`
84+
85+
The secret for the credentials being used.
86+
87+
include::redpanda-connect:components:partial$secret_warning.adoc[]
88+
89+
*Type*: `string`
90+
91+
=== `aws.token`
92+
93+
The token for the credentials being used, required when using short term credentials.
94+
95+
*Type*: `string`
96+
4397
=== `batching`
4498

4599

0 commit comments

Comments
 (0)