DOC-1220 Add Franz-Go timeout fields (#221)

Author: asimms41

modules/components/pages/inputs/kafka_franz.adoc

@@ -56,14 +56,16 @@ input:
       client_certs: [] # No default (optional)
     sasl: [] # No default (optional)
     metadata_max_age: 5m
+    request_timeout_overhead: 10s
+    conn_idle_timeout: 20s
     topics: [] # No default (required)
     regexp_topics: false
     rack_id: "" # No default (optional)
     instance_id: "" No default (optional)
     rebalance_timeout: 45s
     session_timeout: 1m
     heartbeat_interval: 3s
-    start_from_oldest: true
+    start_offset: earliest
     fetch_max_bytes: 50MiB
     fetch_max_wait: 5s
     fetch_min_bytes: 1B
@@ -477,6 +479,22 @@ The maximum period of time after which metadata is refreshed.
 
 *Default*: `5m`
 
+=== `request_timeout_overhead`
+
+Grants an additional buffer or overhead to requests that have timeout fields defined. This field is based on the behavior of Apache Kafka's `request.timeout.ms` parameter.
+
+*Type*: `string`
+
+*Default*: `10s`
+
+=== `conn_idle_timeout`
+
+Define how long connections can remain idle before they are closed.
+
+*Type*: `string`
+
+*Default*: `20s`
+
 === `topics`
 
 A list of topics to consume from. Use commas to separate multiple topics in a single element.
@@ -485,7 +503,7 @@ When a `consumer_group` is specified, partitions are automatically distributed a
 
 Alternatively, you can specify explicit partitions to consume by using a colon after the topic name. For example, `foo:0` would consume the partition `0` of the topic foo. This syntax supports ranges. For example, `foo:0-10` would consume partitions `0` through to `10` inclusive.
 
-It is also possible to specify an explicit offset to consume from by adding another colon after the partition. For example, `foo:0:10` would consume the partition `0` of the topic `foo` starting from the offset `10`. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.
+It is also possible to specify an explicit offset to consume from by adding another colon after the partition. For example, `foo:0:10` would consume the partition `0` of the topic `foo` starting from the offset `10`. If the offset is not present (or remains unspecified) then the field `start_offset` determines which offset to start from.
 
 *Type*: `array`
 
@@ -581,13 +599,29 @@ client
 
 *Default*: `3s`
 
-=== `start_from_oldest`
+=== `start_offset`
 
-Whether to consume from the oldest available offset. Otherwise, messages are consumed from the latest offset. This setting is applied when creating a new consumer group or the saved offset no longer exists.
+Specify the offset from which this input starts or restarts consuming messages. Restarts occur when the `OffsetOutOfRange` error is seen during a fetch.
 
-*Type*: `bool`
+*Type*: `string`
+
+*Default*: `earliest`
+
+Options: 
+
+|===
+| Option | Description
+
+| `earliest` (default)
+| Start from the earliest offset. This option is equivalent to Kafka's `auto.offset.reset=earliest`.
+
+| `committed`
+| Start from the earliest committed offset, which prevents message consumption from a partition in a consumer group that has no prior commits. This option is equivalent to Kafka's `auto.offset.reset=none`.
+
+| `latest`
+| Start from the latest offset. This option is equivalent to Kafka's `auto.offset.reset=latest`.
+|===
 
-*Default*: `true`
 
 === `fetch_max_bytes`
 

modules/components/pages/inputs/ockam_kafka.adoc

@@ -67,7 +67,7 @@ input:
       rebalance_timeout: 45s
       session_timeout: 1m
       heartbeat_interval: 3s
-      start_from_oldest: true
+      start_offset: earliest
       fetch_max_bytes: 50MiB
       fetch_max_wait: 5s
       fetch_min_bytes: 1B
@@ -261,7 +261,7 @@ If you specify a `consumer_group`, partitions are automatically distributed acro
 
 Alternatively, add a colon after the topic name to set the explicit partitions to consume. For example, `foo:0` consumes the partition `0` of the topic `foo`. This syntax also supports ranges. For example, `foo:0-10` consumes all partitions from `0` through to `10` inclusively.
 
-Finally, add another colon after the partition to set an explicit offset to consume from. For example, `foo:0:10` consumes the partition `0` of the topic `foo` starting from the offset `10`. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.
+Finally, add another colon after the partition to set an explicit offset to consume from. For example, `foo:0:10` consumes the partition `0` of the topic `foo` starting from the offset `10`. If the offset is not present (or remains unspecified) then the field `start_offset` determines which offset to start from.
 
 *Type*: `array`
 
@@ -356,13 +356,28 @@ client
 
 *Default*: `3s`
 
-=== `kafka.start_from_oldest`
+=== `kafka.start_offset`
 
-Whether to consume from the oldest available offset. Otherwise, messages are consumed from the latest offset. This setting is applied when creating a new consumer group or the saved offset no longer exists.
+Specify the offset from which this input starts or restarts consuming messages. Restarts occur when the `OffsetOutOfRange` error is seen during a fetch.
 
-*Type*: `bool`
+*Type*: `string`
+
+*Default*: `earliest`
+
+Options: 
+
+|===
+| Option | Description
+
+| `earliest` (default)
+| Start from the earliest offset. This option is equivalent to Kafka's `auto.offset.reset=earliest`.
 
-*Default*: `true`
+| `committed`
+| Start from the earliest committed offset, which prevents message consumption from a partition in a consumer group that has no prior commits. This option is equivalent to Kafka's `auto.offset.reset=none`.
+
+| `latest`
+| Start from the latest offset. This option is equivalent to Kafka's `auto.offset.reset=latest`.
+|===
 
 === `kafka.fetch_max_bytes`
 

modules/components/pages/inputs/redpanda.adoc

@@ -39,7 +39,7 @@ Advanced::
 --
 
 ```yml
-# All config fields, showing default values
+# All configuration fields, showing default values
 input:
   label: "" # No default (optional)
   redpanda:
@@ -54,14 +54,16 @@ input:
       client_certs: [] # No default (optional)
     sasl: [] # No default (optional)
     metadata_max_age: 5m
+    request_timeout_overhead: 10s
+    conn_idle_timeout: 20s
     topics: [] # No default (required)
     regexp_topics: false
     rack_id: "" # No default (optional)
     instance_id: "" # No default (optional)
     rebalance_timeout: 45s
     session_timeout: 1m
     heartbeat_interval: 3s
-    start_from_oldest: true
+    start_offset: earliest
     fetch_max_bytes: 50MiB
     fetch_max_wait: 5s
     fetch_min_bytes: 1B
@@ -495,6 +497,22 @@ The maximum period of time after which metadata is refreshed.
 
 *Default*: `5m`
 
+=== `request_timeout_overhead`
+
+Grants an additional buffer or overhead to requests that have timeout fields defined. This field is based on the behavior of Apache Kafka's `request.timeout.ms` parameter.
+
+*Type*: `string`
+
+*Default*: `10s`
+
+=== `conn_idle_timeout`
+
+Define how long connections can remain idle before they are closed.
+
+*Type*: `string`
+
+*Default*: `20s`
+
 === `topics`
 
 A list of topics to consume from. Use commas to separate multiple topics in a single element.
@@ -503,7 +521,7 @@ When a `consumer_group` is specified, partitions are automatically distributed a
 
 Alternatively, you can specify explicit partitions to consume by using a colon after the topic name. For example, `foo:0` would consume the partition `0` of the topic foo. This syntax supports ranges. For example, `foo:0-10` would consume partitions `0` through to `10` inclusive.
 
-It is also possible to specify an explicit offset to consume from by adding another colon after the partition. For example, `foo:0:10` would consume the partition `0` of the topic `foo` starting from the offset `10`. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.
+It is also possible to specify an explicit offset to consume from by adding another colon after the partition. For example, `foo:0:10` would consume the partition `0` of the topic `foo` starting from the offset `10`. If the offset is not present (or remains unspecified) then the field `start_offset` determines which offset to start from.
 
 
 *Type*: `array`
@@ -599,13 +617,28 @@ client
 
 *Default*: `3s`
 
-=== `start_from_oldest`
+=== `start_offset`
 
-Determines whether to consume from the oldest available offset. Otherwise, messages are consumed from the latest offset. This setting is applied when creating a new consumer group or the saved offset no longer exists.
+Specify the offset from which this input starts or restarts consuming messages. Restarts occur when the `OffsetOutOfRange` error is seen during a fetch.
 
-*Type*: `bool`
+*Type*: `string`
 
-*Default*: `true`
+*Default*: `earliest`
+
+Options: 
+
+|===
+| Option | Description
+
+| `earliest` (default)
+| Start from the earliest offset. This option is equivalent to Kafka's `auto.offset.reset=earliest`.
+
+| `committed`
+| Start from the earliest committed offset, which prevents message consumption from a partition in a consumer group that has no prior commits. This option is equivalent to Kafka's `auto.offset.reset=none`.
+
+| `latest`
+| Start from the latest offset. This option is equivalent to Kafka's `auto.offset.reset=latest`.
+|===
 
 === `fetch_max_bytes`
 

modules/components/pages/inputs/redpanda_common.adoc

@@ -51,7 +51,7 @@ input:
     rebalance_timeout: 45s
     session_timeout: 1m
     heartbeat_interval: 3s
-    start_from_oldest: true
+    start_offset: earliest
     fetch_max_bytes: 50MiB
     fetch_max_wait: 5s
     fetch_min_bytes: 1B
@@ -171,7 +171,7 @@ When a `consumer_group` is specified, partitions are automatically distributed a
 
 Alternatively, you can specify explicit partitions to consume by using a colon after the topic name. For example, `foo:0` would consume the partition `0` of the topic foo. This syntax supports ranges. For example, `foo:0-10` would consume partitions `0` through to `10` inclusive.
 
-It is also possible to specify an explicit offset to consume from by adding another colon after the partition. For example, `foo:0:10` would consume the partition `0` of the topic `foo` starting from the offset `10`. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.
+It is also possible to specify an explicit offset to consume from by adding another colon after the partition. For example, `foo:0:10` would consume the partition `0` of the topic `foo` starting from the offset `10`. If the offset is not present (or remains unspecified) then the field `start_offset` determines which offset to start from.
 
 *Type*: `array`
 
@@ -266,13 +266,28 @@ client
 
 *Default*: `3s`
 
-=== `start_from_oldest`
+=== `start_offset`
 
-Whether to consume from the oldest available offset. Otherwise, messages are consumed from the latest offset. This setting is applied when creating a new consumer group or the saved offset no longer exists.
+Specify the offset from which this input starts or restarts consuming messages. Restarts occur when the `OffsetOutOfRange` error is seen during a fetch.
 
-*Type*: `bool`
+*Type*: `string`
 
-*Default*: `true`
+*Default*: `earliest`
+
+Options: 
+
+|===
+| Option | Description
+
+| `earliest` (default)
+| Start from the earliest offset. This option is equivalent to Kafka's `auto.offset.reset=earliest`.
+
+| `committed`
+| Start from the earliest committed offset, which prevents message consumption from a partition in a consumer group that has no prior commits. This option is equivalent to Kafka's `auto.offset.reset=none`.
+
+| `latest`
+| Start from the latest offset. This option is equivalent to Kafka's `auto.offset.reset=latest`.
+|===
 
 === `fetch_max_bytes`
 

modules/components/pages/inputs/redpanda_migrator.adoc

@@ -63,7 +63,7 @@ input:
     rebalance_timeout: 45s
     session_timeout: 1m
     heartbeat_interval: 3s
-    start_from_oldest: true
+    start_offset: earliest
     fetch_max_bytes: 50MiB
     fetch_max_wait: 5s
     fetch_min_bytes: 1B
@@ -72,6 +72,8 @@ input:
     consumer_group: "" # No default (optional)
     commit_period: 5s
     metadata_max_age: 5m
+    request_timeout_overhead: 10s
+    conn_idle_timeout: 20s
     partition_buffer_bytes: 1MB
     auto_replay_nacks: true
     topic_lag_refresh_period: 5s
@@ -478,7 +480,7 @@ When a `consumer_group` is specified, partitions are automatically distributed a
 
 Alternatively, you can specify explicit partitions to consume by using a colon after the topic name. For example, `foo:0` would consume the partition `0` of the topic foo. This syntax supports ranges. For example, `foo:0-10` would consume partitions `0` through to `10` inclusive.
 
-It is also possible to specify an explicit offset to consume from by adding another colon after the partition. For example, `foo:0:10` would consume the partition `0` of the topic `foo` starting from the offset `10`. If the offset is not present (or remains unspecified) then the field `start_from_oldest` determines which offset to start from.
+It is also possible to specify an explicit offset to consume from by adding another colon after the partition. For example, `foo:0:10` would consume the partition `0` of the topic `foo` starting from the offset `10`. If the offset is not present (or remains unspecified) then the field `start_offset` determines which offset to start from.
 
 
 *Type*: `array`
@@ -576,14 +578,28 @@ client
 
 *Default*: `3s`
 
-=== `start_from_oldest`
+=== `start_offset`
 
-Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. This setting is applied when creating a new consumer group or the saved offset no longer exists.
+Specify the offset from which this input starts or restarts consuming messages. Restarts occur when the `OffsetOutOfRange` error is seen during a fetch.
 
+*Type*: `string`
 
-*Type*: `bool`
+*Default*: `earliest`
 
-*Default*: `true`
+Options: 
+
+|===
+| Option | Description
+
+| `earliest` (default)
+| Start from the earliest offset. This option is equivalent to Kafka's `auto.offset.reset=earliest`.
+
+| `committed`
+| Start from the earliest committed offset, which prevents message consumption from a partition in a consumer group that has no prior commits. This option is equivalent to Kafka's `auto.offset.reset=none`.
+
+| `latest`
+| Start from the latest offset. This option is equivalent to Kafka's `auto.offset.reset=latest`.
+|===
 
 === `fetch_max_bytes`
 
@@ -666,6 +682,22 @@ The maximum period of time after which metadata is refreshed. Reducing this valu
 
 *Default*: `5m`
 
+=== `request_timeout_overhead`
+
+Grants an additional buffer or overhead to requests that have timeout fields defined. This field is based on the behavior of Apache Kafka's `request.timeout.ms` parameter.
+
+*Type*: `string`
+
+*Default*: `10s`
+
+=== `conn_idle_timeout`
+
+Define how long connections can remain idle before they are closed.
+
+*Type*: `string`
+
+*Default*: `20s`
+
 === `partition_buffer_bytes`
 
 A buffer size (in bytes) for each consumed partition, which allows the internal queuing of records before they are flushed. Increasing this value may improve throughput but results in higher memory utilization. 

modules/components/pages/outputs/kafka_franz.adoc

@@ -84,6 +84,8 @@ output:
       client_certs: []
     sasl: [] # No default (optional)
     metadata_max_age: 5m
+    request_timeout_overhead: 10s
+    conn_idle_timeout: 20s
     timestamp_ms: ${! timestamp_unix_milli() } # No default (optional)
 ```
 
@@ -718,6 +720,22 @@ The maximum period of time after which metadata is refreshed.
 
 *Default*: `5m`
 
+=== `request_timeout_overhead`
+
+Grants an additional buffer or overhead to requests that have timeout fields defined. This field is based on the behavior of Apache Kafka's `request.timeout.ms` parameter, but with the option to extend the timeout deadline.
+
+*Type*: `string`
+
+*Default*: `10s`
+
+=== `conn_idle_timeout`
+
+Define how long connections can remain idle before they are closed.
+
+*Type*: `string`
+
+*Default*: `20s`
+
 
 === `timestamp_ms`