diff --git a/content/integrate/redis-data-integration/reference/api-reference/openapi.json b/content/integrate/redis-data-integration/reference/api-reference/openapi.json index 6a676ae1ef..7a1f51b9d2 100644 --- a/content/integrate/redis-data-integration/reference/api-reference/openapi.json +++ b/content/integrate/redis-data-integration/reference/api-reference/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "Redis Data Integration API", "description": "API for Redis Data Integration services", - "version": "1.6.0" + "version": "1.6.1" }, "paths": { "/": { @@ -3114,6 +3114,7 @@ "enum": [ "on_failed_retry_interval", "read_batch_size", + "debezium_lob_encoded_placeholder", "dedup", "dedup_max_size", "dedup_strategy", diff --git a/content/integrate/redis-data-integration/reference/cli/redis-di-delete-all-contexts.md b/content/integrate/redis-data-integration/reference/cli/redis-di-delete-all-contexts.md index 67fd1c3fb7..db46b459f3 100644 --- a/content/integrate/redis-data-integration/reference/cli/redis-di-delete-all-contexts.md +++ b/content/integrate/redis-data-integration/reference/cli/redis-di-delete-all-contexts.md @@ -30,7 +30,7 @@ Usage: redis-di delete-all-contexts [OPTIONS] - Usage: `--force -f` - Force operation. skips verification prompts + Force operation. Skips verification prompts - `help`: @@ -50,6 +50,6 @@ Usage: redis-di delete-all-contexts [OPTIONS] Options: -l, --log-level [TRACE|DEBUG|INFO|WARNING|ERROR|CRITICAL] [default: INFO] - -f, --force Force operation. skips verification prompts + -f, --force Force operation. Skips verification prompts --help Show this message and exit. ``` diff --git a/content/integrate/redis-data-integration/reference/cli/redis-di-delete-context.md b/content/integrate/redis-data-integration/reference/cli/redis-di-delete-context.md index 80bf2ad6a1..110d1f60f1 100644 --- a/content/integrate/redis-data-integration/reference/cli/redis-di-delete-context.md +++ b/content/integrate/redis-data-integration/reference/cli/redis-di-delete-context.md @@ -36,7 +36,7 @@ Usage: redis-di delete-context [OPTIONS] CONTEXT_NAME - Usage: `--force -f` - Force operation. skips verification prompts + Force operation. Skips verification prompts - `help`: @@ -56,6 +56,6 @@ Usage: redis-di delete-context [OPTIONS] CONTEXT_NAME Options: -l, --log-level [TRACE|DEBUG|INFO|WARNING|ERROR|CRITICAL] [default: INFO] - -f, --force Force operation. skips verification prompts + -f, --force Force operation. Skips verification prompts --help Show this message and exit. ``` diff --git a/content/integrate/redis-data-integration/reference/cli/redis-di-get-rejected.md b/content/integrate/redis-data-integration/reference/cli/redis-di-get-rejected.md index 469a86e03a..881b281624 100644 --- a/content/integrate/redis-data-integration/reference/cli/redis-di-get-rejected.md +++ b/content/integrate/redis-data-integration/reference/cli/redis-di-get-rejected.md @@ -102,7 +102,7 @@ Usage: redis-di get-rejected [OPTIONS] - Usage: `--oldest -o` - Displays the oldest rejected records. If omitted, most resent records will be retrieved + Displays the oldest rejected records. If omitted, most recent records will be retrieved - `dlq_name`: @@ -142,7 +142,7 @@ Options: key --max-records INTEGER RANGE Maximum rejected records per DLQ [x>=1] -o, --oldest Displays the oldest rejected records. If - omitted, most resent records will be + omitted, most recent records will be retrieved --dlq-name TEXT Only prints the rejected records for the specified DLQ (Dead Letter Queue) name diff --git a/content/integrate/redis-data-integration/reference/cli/redis-di-reset.md b/content/integrate/redis-data-integration/reference/cli/redis-di-reset.md index c4e958b5d1..fa4c4969cb 100644 --- a/content/integrate/redis-data-integration/reference/cli/redis-di-reset.md +++ b/content/integrate/redis-data-integration/reference/cli/redis-di-reset.md @@ -94,7 +94,7 @@ Usage: redis-di reset [OPTIONS] - Usage: `--force -f` - Force operation. skips verification prompts + Force operation. Skips verification prompts - `pause_for_confirmation`: @@ -132,7 +132,7 @@ Options: --rdi-cacert TEXT CA certificate file to verify with --rdi-key-password TEXT Password for unlocking an encrypted private key - -f, --force Force operation. skips verification prompts + -f, --force Force operation. Skips verification prompts --pause-for-confirmation Pause for user confirmation if manual shutdown of collector required --help Show this message and exit. diff --git a/content/integrate/redis-data-integration/reference/cli/redis-di-upgrade.md b/content/integrate/redis-data-integration/reference/cli/redis-di-upgrade.md index 05ee2acb7c..2dd593f984 100644 --- a/content/integrate/redis-data-integration/reference/cli/redis-di-upgrade.md +++ b/content/integrate/redis-data-integration/reference/cli/redis-di-upgrade.md @@ -102,7 +102,7 @@ Usage: redis-di upgrade [OPTIONS] - Usage: `--force -f` - Force operation. skips verification prompts + Force operation. Skips verification prompts - `help`: @@ -133,6 +133,6 @@ Options: --rdi-cacert TEXT CA certificate file to verify with --rdi-key-password TEXT Password for unlocking an encrypted private key - -f, --force Force operation. skips verification prompts + -f, --force Force operation. Skips verification prompts --help Show this message and exit. ``` diff --git a/content/integrate/redis-data-integration/reference/config-yaml-reference.md b/content/integrate/redis-data-integration/reference/config-yaml-reference.md index 0ae91f2c11..2bc3f03fc4 100644 --- a/content/integrate/redis-data-integration/reference/config-yaml-reference.md +++ b/content/integrate/redis-data-integration/reference/config-yaml-reference.md @@ -17,7 +17,7 @@ Configuration file for Redis Data Integration (RDI) source collectors and target | [**sources**](#sources)
(Source collectors) | `object` | Defines source collectors and their configurations. Each key represents a unique source identifier, and its value contains specific configuration for that collector
| | | [**processors**](#processors)
(Data processing configuration) | `object`, `null` | Configuration settings that control how data is processed, including batch sizes, error handling, and performance tuning
| | | [**targets**](#targets)
(Target connections) | `object` | Configuration for target Redis databases where processed data will be written
| | -| [**secret\-providers**](#secret-providers)
(Secret providers) | `object` | Configuration for secret management providers
| | +| [**secret\-providers**](#secret-providers)
(Secret providers) | `object` | | | @@ -39,26 +39,27 @@ Configuration settings that control how data is processed, including batch sizes **Properties** -| Name | Type | Description | Required | -| -------------------------------------------------------------------- | ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -| **on_failed_retry_interval**
(Retry interval on failure) | `integer`, `string` | Number of seconds to wait before retrying a failed operation
Default: `5`
Pattern: `^\${.*}$`
Minimum: `1`
| | -| **read_batch_size** | `integer`, `string` | Maximum number of records to process in a single batch
Default: `2000`
Pattern: `^\${.*}$`
Minimum: `1`
| | -| **dedup**
(Enable deduplication) | `boolean` | Enable the deduplication mechanism to handle duplicate records
Default: `false`
| | -| **dedup_max_size**
(Deduplication set size) | `integer` | Maximum number of entries to store in the deduplication set
Default: `1024`
Minimum: `1`
| | -| **dedup_strategy**
(Deduplication strategy) | `string` | (DEPRECATED)
Property 'dedup_strategy' is now deprecated. The only supported strategy is 'ignore'. Please remove from the configuration.
Default: `"ignore"`
Enum: `"reject"`, `"ignore"`
| | -| **duration**
(Batch duration limit) | `integer`, `string` | Maximum time in milliseconds to wait for a batch to fill before processing
Default: `100`
Pattern: `^\${.*}$`
Minimum: `1`
| | -| **write_batch_size** | `integer`, `string` | Maximum number of records to write to target Redis database in a single batch
Default: `200`
Pattern: `^\${.*}$`
Minimum: `1`
| | -| **error_handling**
(Error handling strategy) | `string` | Strategy for handling errors: ignore to skip errors, dlq to store rejected messages in dead letter queue
Default: `"dlq"`
Pattern: `^\${.*}$\|ignore\|dlq`
| | -| **dlq_max_messages**
(DLQ message limit) | `integer`, `string` | Maximum number of messages to store in dead letter queue per stream
Default: `1000`
Pattern: `^\${.*}$`
Minimum: `1`
| | -| **target_data_type**
(Target Redis data type) | `string` | Data type to use in Redis: hash for Redis Hash, json for RedisJSON (requires RedisJSON module)
Default: `"hash"`
Pattern: `^\${.*}$\|hash\|json`
| | -| **json_update_strategy** | `string` | Strategy for updating JSON data in Redis: replace to overwrite the entire JSON object, merge to merge new data with existing JSON object
Default: `"replace"`
Pattern: `^\${.*}$\|replace\|merge`
| | -| **initial_sync_processes** | `integer`, `string` | Number of parallel processes for performing initial data synchronization
Default: `4`
Pattern: `^\${.*}$`
Minimum: `1`
Maximum: `32`
| | -| **idle_sleep_time_ms**
(Idle sleep interval) | `integer`, `string` | Time in milliseconds to sleep between processing batches when idle
Default: `200`
Pattern: `^\${.*}$`
Minimum: `1`
Maximum: `999999`
| | -| **idle_streams_check_interval_ms**
(Idle streams check interval) | `integer`, `string` | Time in milliseconds between checking for new streams when processor is idle
Default: `1000`
Pattern: `^\${.*}$`
Minimum: `1`
Maximum: `999999`
| | -| **busy_streams_check_interval_ms**
(Busy streams check interval) | `integer`, `string` | Time in milliseconds between checking for new streams when processor is busy
Default: `5000`
Pattern: `^\${.*}$`
Minimum: `1`
Maximum: `999999`
| | -| **wait_enabled**
(Enable replica wait) | `boolean` | Enable verification that data has been written to replica shards
Default: `false`
| | -| **wait_timeout**
(Replica wait timeout) | `integer`, `string` | Maximum time in milliseconds to wait for replica write verification
Default: `1000`
Pattern: `^\${.*}$`
Minimum: `1`
| | -| **retry_on_replica_failure** | `boolean` | Continue retrying writes until successful replication to replica shards is confirmed
Default: `true`
| | +| Name | Type | Description | Required | +| -------------------------------------------------------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | +| **on_failed_retry_interval**
(Retry interval on failure) | `integer`, `string` | Number of seconds to wait before retrying a failed operation
Default: `5`
Pattern: `^\${.*}$`
Minimum: `1`
| | +| **read_batch_size** | `integer`, `string` | Maximum number of records to process in a single batch
Default: `2000`
Pattern: `^\${.*}$`
Minimum: `1`
| | +| **debezium_lob_encoded_placeholder**
(Debezium LOB placeholder) | `string` | Placeholder value for LOB fields in Debezium
Default: `"__debezium_unavailable_value"`
| | +| **dedup**
(Enable deduplication) | `boolean` | Enable the deduplication mechanism to handle duplicate records
Default: `false`
| | +| **dedup_max_size**
(Deduplication set size) | `integer` | Maximum number of entries to store in the deduplication set
Default: `1024`
Minimum: `1`
| | +| **dedup_strategy**
(Deduplication strategy) | `string` | (DEPRECATED)
Property 'dedup_strategy' is now deprecated. The only supported strategy is 'ignore'. Please remove from the configuration.
Default: `"ignore"`
Enum: `"reject"`, `"ignore"`
| | +| **duration**
(Batch duration limit) | `integer`, `string` | Maximum time in milliseconds to wait for a batch to fill before processing
Default: `100`
Pattern: `^\${.*}$`
Minimum: `1`
| | +| **write_batch_size** | `integer`, `string` | Maximum number of records to write to target Redis database in a single batch
Default: `200`
Pattern: `^\${.*}$`
Minimum: `1`
| | +| **error_handling**
(Error handling strategy) | `string` | Strategy for handling errors: ignore to skip errors, dlq to store rejected messages in dead letter queue
Default: `"dlq"`
Pattern: `^\${.*}$\|ignore\|dlq`
| | +| **dlq_max_messages**
(DLQ message limit) | `integer`, `string` | Maximum number of messages to store in dead letter queue per stream
Default: `1000`
Pattern: `^\${.*}$`
Minimum: `1`
| | +| **target_data_type**
(Target Redis data type) | `string` | Data type to use in Redis: hash for Redis Hash, json for RedisJSON (requires RedisJSON module)
Default: `"hash"`
Pattern: `^\${.*}$\|hash\|json`
| | +| **json_update_strategy** | `string` | (DEPRECATED)
Property 'json_update_strategy' will be deprecated in future releases. Use 'on_update' job-level property to define the json update strategy.
Default: `"replace"`
Pattern: `^\${.*}$\|replace\|merge`
| | +| **initial_sync_processes** | `integer`, `string` | Number of parallel processes for performing initial data synchronization
Default: `4`
Pattern: `^\${.*}$`
Minimum: `1`
Maximum: `32`
| | +| **idle_sleep_time_ms**
(Idle sleep interval) | `integer`, `string` | Time in milliseconds to sleep between processing batches when idle
Default: `200`
Pattern: `^\${.*}$`
Minimum: `1`
Maximum: `999999`
| | +| **idle_streams_check_interval_ms**
(Idle streams check interval) | `integer`, `string` | Time in milliseconds between checking for new streams when processor is idle
Default: `1000`
Pattern: `^\${.*}$`
Minimum: `1`
Maximum: `999999`
| | +| **busy_streams_check_interval_ms**
(Busy streams check interval) | `integer`, `string` | Time in milliseconds between checking for new streams when processor is busy
Default: `5000`
Pattern: `^\${.*}$`
Minimum: `1`
Maximum: `999999`
| | +| **wait_enabled**
(Enable replica wait) | `boolean` | Enable verification that data has been written to replica shards
Default: `false`
| | +| **wait_timeout**
(Replica wait timeout) | `integer`, `string` | Maximum time in milliseconds to wait for replica write verification
Default: `1000`
Pattern: `^\${.*}$`
Minimum: `1`
| | +| **retry_on_replica_failure** | `boolean` | Continue retrying writes until successful replication to replica shards is confirmed
Default: `true`
| | **Additional Properties:** not allowed @@ -77,8 +78,6 @@ Configuration for target Redis databases where processed data will be written ## secret\-providers: Secret providers -Configuration for secret management providers - **Properties (Pattern)** | Name | Type | Description | Required | @@ -91,10 +90,10 @@ Configuration for secret management providers **Properties** -| Name | Type | Description | Required | -| ----------------------------------------------------------------------- | -------- | ----------------------------------------------------------------- | -------- | -| **type**
(Provider type) | `string` | Type of secret provider service
Enum: `"aws"`, `"vault"`
| yes | -| [**parameters**](#secret-providersparameters)
(Provider parameters) | `object` | Configuration parameters for the secret provider
| yes | +| Name | Type | Description | Required | +| ----------------------------------------------------------------------- | -------- | ----------------------------- | -------- | +| **type**
(Provider type) | `string` | Enum: `"aws"`, `"vault"`
| yes | +| [**parameters**](#secret-providersparameters)
(Provider parameters) | `object` | | yes | **Additional Properties:** not allowed **Example** @@ -109,13 +108,11 @@ parameters: ##### secret\-providers\.\.\*\.parameters: Provider parameters -Configuration parameters for the secret provider - **Properties** -| Name | Type | Description | Required | -| ----------------------------------------------------------------------------- | ---------- | ------------------------------------------------------ | -------- | -| [**objects**](#secret-providersparametersobjects)
(Secrets objects array) | `object[]` | List of secret objects to fetch from the provider
| yes | +| Name | Type | Description | Required | +| ----------------------------------------------------------------------------- | ---------- | ----------- | -------- | +| [**objects**](#secret-providersparametersobjects)
(Secrets objects array) | `object[]` | | yes | **Example** @@ -128,8 +125,6 @@ objects: ###### secret\-providers\.\.\*\.parameters\.objects\[\]: Secrets objects array -List of secret objects to fetch from the provider - **Items: Secret object** **No properties.**