diff --git a/content/integrate/redis-data-integration/data-pipelines/data-denormalization.md b/content/integrate/redis-data-integration/data-pipelines/data-denormalization.md index 1e598bdfc6..3d4d4a84e2 100644 --- a/content/integrate/redis-data-integration/data-pipelines/data-denormalization.md +++ b/content/integrate/redis-data-integration/data-pipelines/data-denormalization.md @@ -117,7 +117,6 @@ To configure normalization, you must first configure the parent entity to use JS ```yaml # jobs/invoice.yaml source: - server_name: chinook schema: public table: Invoice @@ -138,7 +137,6 @@ After you have configured the parent entity, you can then configure the child en ```yaml # jobs/invoice_line.yaml source: - server_name: chinook schema: public table: InvoiceLine output: @@ -146,7 +144,6 @@ output: with: nest: # cannot co-exist with other parameters such as 'key' parent: - # server_name: chinook # schema: public table: Invoice nesting_key: InvoiceLineId # the unique key in the composite structure under which the child data will be stored @@ -161,10 +158,8 @@ output: The job has a `with` section under `output` that includes the `nest` block. The job must include the following attributes in the `nest` block: -- `parent`: This specifies the RDI data stream for the parent entities. Typically, you only - need to supply the parent `table` name, unless you are nesting children under a parent that comes from - a different source database. If you do this then you must also specify `server_name` and - `schema` attributes. Note that this attribute refers to a Redis *key* that will be added to the target +- `parent`: This specifies the config of the parent entities. You only + need to supply the parent `table` name. Note that this attribute refers to a Redis *key* that will be added to the target database, not to a table you can access from the pipeline. See [Using nesting](#using-nesting) below for the format of the key that is generated. - `nesting_key`: The unique key of each child entry in the JSON map that will be created under the path. diff --git a/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-hash-example.md b/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-hash-example.md index c1be633a33..916709e219 100644 --- a/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-hash-example.md +++ b/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-hash-example.md @@ -24,7 +24,6 @@ If you don't supply an `expire` parameter, the keys will never expire. ```yaml source: - server_name: chinook schema: public table: invoice output: @@ -36,4 +35,4 @@ output: expression: concat(['invoice_id:', InvoiceId]) language: jmespath expire: 100 -``` \ No newline at end of file +``` diff --git a/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-json-example.md b/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-json-example.md index bd812e9189..312a765312 100644 --- a/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-json-example.md +++ b/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-json-example.md @@ -29,7 +29,6 @@ In this case, the result will be Redis JSON documents with key names based on th ```yaml source: - server_name: chinook schema: public table: invoice output: @@ -42,4 +41,4 @@ output: language: jmespath on_update: replace expire: 100 -``` \ No newline at end of file +``` diff --git a/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-set-example.md b/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-set-example.md index 1faa3e11a6..7b4b7c5843 100644 --- a/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-set-example.md +++ b/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-set-example.md @@ -23,7 +23,6 @@ When writing to a set, you must supply an extra argument, `member`, which specif ```yaml source: - server_name: chinook schema: public table: invoice output: @@ -37,4 +36,4 @@ output: args: member: InvoiceId expire: 100 -``` \ No newline at end of file +``` diff --git a/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-set-key-name.md b/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-set-key-name.md index 48fe9aa9ad..a309de2cc7 100644 --- a/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-set-key-name.md +++ b/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-set-key-name.md @@ -47,7 +47,6 @@ To customize key names, use the `key` section within the `redis.write` output co ```yaml source: - server_name: rdi db: inventory table: customers output: @@ -64,7 +63,6 @@ When working with the full row format, you need to handle key generation differe ```yaml source: - server_name: rdi db: inventory table: customers row_format: full diff --git a/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-sorted-set-example.md b/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-sorted-set-example.md index 853fe24921..749afe0013 100644 --- a/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-sorted-set-example.md +++ b/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-sorted-set-example.md @@ -23,7 +23,6 @@ When writing to sorted sets, you must provide two additional arguments, `member` ```yaml source: - server_name: chinook schema: public table: invoice output: @@ -44,4 +43,4 @@ Since sorted sets in Redis are inherently sorted, you can easily get the top N i ``` ZREVRANGE invoices:sorted 0 9 WITHSCORES -``` \ No newline at end of file +``` diff --git a/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-stream-example.md b/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-stream-example.md index 0d110f1d41..bc110f814b 100644 --- a/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-stream-example.md +++ b/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-stream-example.md @@ -29,7 +29,6 @@ In the example, only three original fields are passed in the message payload: `I ```yaml source: - server_name: chinook schema: public table: invoice output: @@ -45,4 +44,4 @@ output: - BillingCountry: country - Total expire: 100 -``` \ No newline at end of file +``` diff --git a/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-string-example.md b/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-string-example.md index c615721269..4f8254d777 100644 --- a/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-string-example.md +++ b/content/integrate/redis-data-integration/data-pipelines/transform-examples/redis-string-example.md @@ -35,7 +35,6 @@ If you don't supply an `expire` parameter, the keys will never expire. ```yaml source: - server_name: chinook table: album row_format: full output: @@ -49,4 +48,4 @@ output: args: value: title expire: 86400 -``` \ No newline at end of file +``` diff --git a/content/integrate/redis-data-integration/reference/api-reference/openapi.json b/content/integrate/redis-data-integration/reference/api-reference/openapi.json index 580b7ae885..6c4808d85e 100644 --- a/content/integrate/redis-data-integration/reference/api-reference/openapi.json +++ b/content/integrate/redis-data-integration/reference/api-reference/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "Redis Data Integration API", "description": "API for Redis Data Integration services", - "version": "1.8.0" + "version": "1.12.0" }, "paths": { "/": { @@ -1814,7 +1814,7 @@ "secure" ], "summary": "Get Metadata Per Schema", - "description": "Fetches metadata for specified schema from a source.\n\nArgs:\n source_name (str): Source name.\n schema (Optional[str]): Schema name. Defaults to None.\n\nReturns:\n MetadataResponse: Detailed metadata information for the specified schema, or all schemas if none specified.", + "description": "Fetches metadata for specified schema from a source.\n\nArgs:\n source_name (str): Source name.\n schema (Optional[str]): Schema name. Defaults to None.\n tables (Optional[str]): Comma-separated list of table names to filter by. Defaults to None.\n\nReturns:\n MetadataResponse: Detailed metadata information for the specified schema and tables, or all schemas/tables if none specified.", "operationId": "get_metadata_per_schema_api_v1_pipelines_sources__source_name__metadata_get", "security": [ { @@ -1846,6 +1846,24 @@ ], "title": "Schema" } + }, + { + "name": "tables", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Comma-separated list of table names to filter by", + "title": "Tables" + }, + "description": "Comma-separated list of table names to filter by" } ], "responses": { @@ -1978,6 +1996,11 @@ "summary": "Retrieve Pipelines", "description": "Retrieves a pipeline.", "operationId": "retrieve_pipelines_api_v1_pipelines_get", + "security": [ + { + "JWTBearer": [] + } + ], "responses": { "200": { "description": "Successful Response", @@ -2010,12 +2033,7 @@ } } } - }, - "security": [ - { - "JWTBearer": [] - } - ] + } }, "post": { "tags": [ @@ -2023,14 +2041,38 @@ "deploy" ], "summary": "Pipelines", - "description": "Deploys a pipeline.", + "description": "Deploys a pipeline.\n\nArgs:\n body: Pipeline configuration\n dry_run: If True, only validates the pipeline without deploying it.", "operationId": "pipelines_api_v1_pipelines_post", + "security": [ + { + "JWTBearer": [] + } + ], + "parameters": [ + { + "name": "dry_run", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "title": "Dry Run" + } + } + ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "type": "object", - "title": "Body", "examples": [ { "sources": { @@ -2208,7 +2250,6 @@ { "name": "chinook_customer", "source": { - "server_name": "chinook", "schema": "public", "table": "Customer" }, @@ -2228,11 +2269,11 @@ } ] } - ] + ], + "title": "Body" } } - }, - "required": true + } }, "responses": { "200": { @@ -2240,15 +2281,7 @@ "content": { "application/json": { "schema": { - "anyOf": [ - { - "$ref": "#/components/schemas/ActionIdResponse" - }, - { - "$ref": "#/components/schemas/redis_di_api__v1__modules__shared__response__Error" - } - ], - "title": "Response Pipelines Api V1 Pipelines Post" + "$ref": "#/components/schemas/ActionIdResponse" } } } @@ -2274,21 +2307,16 @@ } }, "422": { - "description": "Unprocessable Entity", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/redis_di_api__v1__modules__shared__response__Error" + "$ref": "#/components/schemas/ApiErrorDetail" } } - } - } - }, - "security": [ - { - "JWTBearer": [] + }, + "description": "Unprocessable Entity" } - ] + } }, "patch": { "tags": [ @@ -2296,14 +2324,38 @@ "deploy" ], "summary": "Patch Pipelines", - "description": "This endpoint allows for partial updates to the pipeline configuration.It expects a JSON Object containing pipeline configuration keys to be updated.JSON Merge Patch is used to update the configuration, see RFC 7396. Arrays are replaced, not merged.", + "description": "This endpoint allows for partial updates to the pipeline configuration. It expects a JSON Object containing pipeline configuration keys to be updated. JSON Merge Patch is used to update the configuration, see RFC 7396. Arrays are replaced, not merged.", "operationId": "patch_pipelines_api_v1_pipelines_patch", + "security": [ + { + "JWTBearer": [] + } + ], + "parameters": [ + { + "name": "dry_run", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "title": "Dry Run" + } + } + ], "requestBody": { + "required": true, "content": { "application/json": { "schema": { "type": "object", - "title": "Body", "examples": [ { "sources": { @@ -2481,7 +2533,6 @@ { "name": "chinook_customer", "source": { - "server_name": "chinook", "schema": "public", "table": "Customer" }, @@ -2514,11 +2565,11 @@ "target_data_type": "hash" } } - ] + ], + "title": "Body" } } - }, - "required": true + } }, "responses": { "200": { @@ -2552,21 +2603,16 @@ } }, "422": { - "description": "Unprocessable Entity", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/redis_di_api__v1__modules__shared__response__Error" + "$ref": "#/components/schemas/ApiErrorDetail" } } - } - } - }, - "security": [ - { - "JWTBearer": [] + }, + "description": "Unprocessable Entity" } - ] + } } }, "/api/v1/pipelines/undeploy": { @@ -2678,7 +2724,7 @@ "secure" ], "summary": "Pipelines Config Templates", - "description": "Returns the YAML template describing the config for the specifieddb type.", + "description": "Returns the YAML template describing the config for the specified db type.", "operationId": "pipelines_config_templates_api_v1_pipelines_config_templates_ingest__db_type__get", "security": [ { @@ -2693,6 +2739,24 @@ "schema": { "$ref": "#/components/schemas/DbType" } + }, + { + "name": "db_flavor", + "in": "query", + "required": false, + "schema": { + "anyOf": [ + { + "$ref": "#/components/schemas/DbFlavor" + }, + { + "type": "null" + } + ], + "description": "Optional database flavor", + "title": "Db Flavor" + }, + "description": "Optional database flavor" } ], "responses": { @@ -3542,7 +3606,14 @@ "ActionIdResponse": { "properties": { "action_id": { - "type": "string", + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], "title": "Action Id", "examples": [ "1234567890" @@ -3550,9 +3621,6 @@ } }, "type": "object", - "required": [ - "action_id" - ], "title": "ActionIdResponse", "description": "Response model containing an action ID." }, @@ -3614,8 +3682,22 @@ "type": "string", "title": "Detail", "examples": [ - "Incorrect username or password" + "Detailed error message" ] + }, + "errors": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/redis_di_api__models__errors__Error" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Errors" } }, "type": "object", @@ -4088,6 +4170,16 @@ "title": "DatabaseResponse", "description": "Database response class." }, + "DbFlavor": { + "type": "string", + "enum": [ + "mongodb-atlas", + "mongodb-replica-set", + "mongodb-sharded-cluster" + ], + "title": "DbFlavor", + "description": "Defines supported database flavors." + }, "DbType": { "type": "string", "enum": [ @@ -4161,6 +4253,15 @@ ], "title": "Entity" }, + "ErrorCode": { + "type": "string", + "enum": [ + "validation_error", + "job_validation_error" + ], + "title": "ErrorCode", + "description": "Error codes" + }, "HTTPValidationError": { "properties": { "detail": { @@ -4765,20 +4866,6 @@ }, "Source": { "properties": { - "server_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "title": "Server Name", - "examples": [ - "localhost" - ] - }, "schema": { "anyOf": [ { @@ -5435,6 +5522,29 @@ ], "title": "WithInput" }, + "redis_di_api__models__errors__Error": { + "properties": { + "code": { + "$ref": "#/components/schemas/ErrorCode" + }, + "message": { + "type": "string", + "title": "Message" + }, + "details": { + "type": "object", + "title": "Details", + "default": {} + } + }, + "type": "object", + "required": [ + "code", + "message" + ], + "title": "Error", + "description": "Generic error model" + }, "redis_di_api__v1__modules__shared__response__Error": { "properties": { "status": { diff --git a/content/integrate/redis-data-integration/reference/data-transformation/add_field.md b/content/integrate/redis-data-integration/reference/data-transformation/add_field.md index b70c7530b3..fee1268cc2 100644 --- a/content/integrate/redis-data-integration/reference/data-transformation/add_field.md +++ b/content/integrate/redis-data-integration/reference/data-transformation/add_field.md @@ -34,7 +34,6 @@ Add multiple fields ```yaml source: - server_name: redislabs schema: dbo table: emp transform: @@ -66,7 +65,6 @@ Add one field ```yaml source: - server_name: redislabs schema: dbo table: emp transform: diff --git a/content/integrate/redis-data-integration/reference/data-transformation/filter.md b/content/integrate/redis-data-integration/reference/data-transformation/filter.md index 704feb0652..17df9de484 100644 --- a/content/integrate/redis-data-integration/reference/data-transformation/filter.md +++ b/content/integrate/redis-data-integration/reference/data-transformation/filter.md @@ -32,7 +32,6 @@ Filter records ```yaml source: - server_name: redislabs schema: dbo table: emp transform: diff --git a/content/integrate/redis-data-integration/reference/data-transformation/map.md b/content/integrate/redis-data-integration/reference/data-transformation/map.md index afb06ef9c1..8691bf5e98 100644 --- a/content/integrate/redis-data-integration/reference/data-transformation/map.md +++ b/content/integrate/redis-data-integration/reference/data-transformation/map.md @@ -32,7 +32,6 @@ Map a record into a new output based on expressions ```yaml source: - server_name: redislabs schema: dbo table: emp transform: diff --git a/content/integrate/redis-data-integration/reference/data-transformation/remove_field.md b/content/integrate/redis-data-integration/reference/data-transformation/remove_field.md index 51937f8e0a..84e780a731 100644 --- a/content/integrate/redis-data-integration/reference/data-transformation/remove_field.md +++ b/content/integrate/redis-data-integration/reference/data-transformation/remove_field.md @@ -34,7 +34,6 @@ Remove multiple fields ```yaml source: - server_name: redislabs schema: dbo table: emp transform: @@ -59,7 +58,6 @@ Remove one field ```yaml source: - server_name: redislabs schema: dbo table: emp transform: diff --git a/content/integrate/redis-data-integration/reference/data-transformation/rename_field.md b/content/integrate/redis-data-integration/reference/data-transformation/rename_field.md index 6fef334bee..33456d3f53 100644 --- a/content/integrate/redis-data-integration/reference/data-transformation/rename_field.md +++ b/content/integrate/redis-data-integration/reference/data-transformation/rename_field.md @@ -33,7 +33,6 @@ Rename multiple fields ```yaml source: - server_name: redislabs schema: dbo table: emp transform: @@ -62,7 +61,6 @@ Rename one field ```yaml source: - server_name: redislabs schema: dbo table: emp transform: