From 7f3a5a5a529f4ecd6247ba05ee86b636f6aa60cc Mon Sep 17 00:00:00 2001 From: shortcuts Date: Fri, 16 May 2025 17:14:37 +0200 Subject: [PATCH 1/6] feat(specs): add global `push` endpoint --- specs/ingestion/common/parameters.yml | 8 ++++++ specs/ingestion/common/schemas/task.yml | 21 ++++++++++++++ specs/ingestion/paths/push.yml | 31 +++++++++++++++++++++ specs/ingestion/paths/tasks/v2/pushTask.yml | 27 ++---------------- specs/ingestion/spec.yml | 3 ++ 5 files changed, 65 insertions(+), 25 deletions(-) create mode 100644 specs/ingestion/paths/push.yml diff --git a/specs/ingestion/common/parameters.yml b/specs/ingestion/common/parameters.yml index d6bdfe9e465..62e4a3ee873 100644 --- a/specs/ingestion/common/parameters.yml +++ b/specs/ingestion/common/parameters.yml @@ -83,3 +83,11 @@ orderKeys: description: Ascending or descending sort order. default: desc enum: [asc, desc] + +watch: + name: watch + in: query + description: When provided, the push operation will be synchronous and the API will wait for the ingestion to be finished before responding. + required: false + schema: + type: boolean diff --git a/specs/ingestion/common/schemas/task.yml b/specs/ingestion/common/schemas/task.yml index 4c0817646dc..fd9ebefc7b3 100644 --- a/specs/ingestion/common/schemas/task.yml +++ b/specs/ingestion/common/schemas/task.yml @@ -606,3 +606,24 @@ Policies: type: integer minimum: 1 maximum: 10 + +PushTaskPayload: + title: pushTaskPayload + type: object + properties: + action: + $ref: '../../../../common/schemas/Batch.yml#/action' + records: + type: array + items: + title: pushTaskRecords + type: object + additionalProperties: true + required: + - objectID + properties: + objectID: + $ref: '../../../../common/parameters.yml#/objectID' + required: + - action + - records diff --git a/specs/ingestion/paths/push.yml b/specs/ingestion/paths/push.yml new file mode 100644 index 00000000000..b8d3aa422ce --- /dev/null +++ b/specs/ingestion/paths/push.yml @@ -0,0 +1,31 @@ +post: + summary: Push a `batch` request payload through the Pipeline + description: Push a `batch` request payload through the Pipeline. You can check the status of your request with the observability endpoints. + operationId: push + x-acl: + - addObject + - deleteIndex + - editSettings + x-timeouts: + connect: 180000 + read: 180000 + write: 180000 + parameters: + - $ref: '../../common/parameters.yml#/IndexName' + - $ref: '../common/parameters.yml#/watch' + requestBody: + description: Request body of a Search API `batch` request that will be pushed in the Connectors pipeline. + content: + application/json: + schema: + $ref: '../common/schemas/task.yml#/PushTaskPayload' + required: true + responses: + '200': + description: OK + content: + application/json: + schema: + $ref: '../../../common/schemas/common.yml#/WatchResponse' + '400': + $ref: '../../../../common/responses/BadRequest.yml' diff --git a/specs/ingestion/paths/tasks/v2/pushTask.yml b/specs/ingestion/paths/tasks/v2/pushTask.yml index b689a9721d8..c01107b17ec 100644 --- a/specs/ingestion/paths/tasks/v2/pushTask.yml +++ b/specs/ingestion/paths/tasks/v2/pushTask.yml @@ -14,36 +14,13 @@ post: write: 180000 parameters: - $ref: '../../../common/parameters.yml#/pathTaskID' - - name: watch - in: query - description: When provided, the push operation will be synchronous and the API will wait for the ingestion to be finished before responding. - required: false - schema: - type: boolean + - $ref: '../../../common/parameters.yml#/watch' requestBody: description: Request body of a Search API `batch` request that will be pushed in the Connectors pipeline. content: application/json: schema: - title: pushTaskPayload - type: object - properties: - action: - $ref: '../../../../common/schemas/Batch.yml#/action' - records: - type: array - items: - title: pushTaskRecords - type: object - additionalProperties: true - required: - - objectID - properties: - objectID: - $ref: '../../../../common/parameters.yml#/objectID' - required: - - action - - records + $ref: '../../../common/schemas/task.yml#/PushTaskPayload' required: true responses: '200': diff --git a/specs/ingestion/spec.yml b/specs/ingestion/spec.yml index 9f42fe8ac9d..9d7a3cb6825 100644 --- a/specs/ingestion/spec.yml +++ b/specs/ingestion/spec.yml @@ -112,6 +112,9 @@ paths: /{path}: $ref: '../common/paths/customRequest.yml' + /1/push/{indexName}: + $ref: 'paths/push.yml' + # authentications API. /1/authentications: $ref: 'paths/authentications/authentications.yml' From 7f69d6e822e49bf313ef5ed1f7c464ddf7996ee5 Mon Sep 17 00:00:00 2001 From: shortcuts Date: Fri, 16 May 2025 17:15:41 +0200 Subject: [PATCH 2/6] chore: tests --- tests/CTS/requests/ingestion/push.json | 85 ++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 tests/CTS/requests/ingestion/push.json diff --git a/tests/CTS/requests/ingestion/push.json b/tests/CTS/requests/ingestion/push.json new file mode 100644 index 00000000000..a3674510a45 --- /dev/null +++ b/tests/CTS/requests/ingestion/push.json @@ -0,0 +1,85 @@ +[ + { + "parameters": { + "indexName": "foo", + "pushTaskPayload": { + "action": "addObject", + "records": [ + { + "key": "bar", + "foo": "1", + "objectID": "o" + }, + { + "key": "baz", + "foo": "2", + "objectID": "k" + } + ] + } + }, + "request": { + "path": "/1/push/foo", + "method": "POST", + "body": { + "action": "addObject", + "records": [ + { + "key": "bar", + "foo": "1", + "objectID": "o" + }, + { + "key": "baz", + "foo": "2", + "objectID": "k" + } + ] + } + } + }, + { + "testName": "allows for watch query parameter", + "parameters": { + "indexName": "bar", + "pushTaskPayload": { + "action": "addObject", + "records": [ + { + "key": "bar", + "foo": "1", + "objectID": "o" + }, + { + "key": "baz", + "foo": "2", + "objectID": "k" + } + ] + }, + "watch": true + }, + "request": { + "path": "/1/push/bar", + "method": "POST", + "queryParameters": { + "watch": "true" + }, + "body": { + "action": "addObject", + "records": [ + { + "key": "bar", + "foo": "1", + "objectID": "o" + }, + { + "key": "baz", + "foo": "2", + "objectID": "k" + } + ] + } + } + } +] From f472aac2077ee566789bb1750b78179c7d937d0b Mon Sep 17 00:00:00 2001 From: shortcuts Date: Fri, 16 May 2025 17:22:30 +0200 Subject: [PATCH 3/6] chore: specs --- specs/ingestion/common/schemas/task.yml | 4 ++-- specs/ingestion/paths/push.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/ingestion/common/schemas/task.yml b/specs/ingestion/common/schemas/task.yml index fd9ebefc7b3..158b7f4e8b9 100644 --- a/specs/ingestion/common/schemas/task.yml +++ b/specs/ingestion/common/schemas/task.yml @@ -612,7 +612,7 @@ PushTaskPayload: type: object properties: action: - $ref: '../../../../common/schemas/Batch.yml#/action' + $ref: '../../../common/schemas/Batch.yml#/action' records: type: array items: @@ -623,7 +623,7 @@ PushTaskPayload: - objectID properties: objectID: - $ref: '../../../../common/parameters.yml#/objectID' + $ref: '../../../common/parameters.yml#/objectID' required: - action - records diff --git a/specs/ingestion/paths/push.yml b/specs/ingestion/paths/push.yml index b8d3aa422ce..e0ddf303f3c 100644 --- a/specs/ingestion/paths/push.yml +++ b/specs/ingestion/paths/push.yml @@ -26,6 +26,6 @@ post: content: application/json: schema: - $ref: '../../../common/schemas/common.yml#/WatchResponse' + $ref: '../common/schemas/common.yml#/WatchResponse' '400': - $ref: '../../../../common/responses/BadRequest.yml' + $ref: '../../common/responses/BadRequest.yml' From 37c33aa07c80e2357e6eb03144e594eaac43471b Mon Sep 17 00:00:00 2001 From: shortcuts Date: Fri, 16 May 2025 17:30:50 +0200 Subject: [PATCH 4/6] chore: test name --- tests/CTS/requests/ingestion/push.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/CTS/requests/ingestion/push.json b/tests/CTS/requests/ingestion/push.json index a3674510a45..9858fef2bd1 100644 --- a/tests/CTS/requests/ingestion/push.json +++ b/tests/CTS/requests/ingestion/push.json @@ -1,5 +1,6 @@ [ { + "testName": "global push", "parameters": { "indexName": "foo", "pushTaskPayload": { @@ -39,7 +40,7 @@ } }, { - "testName": "allows for watch query parameter", + "testName": "global push with watch mode", "parameters": { "indexName": "bar", "pushTaskPayload": { From 45cf97500032a3d4525f5e87a723e7f53cddc9f1 Mon Sep 17 00:00:00 2001 From: shortcuts Date: Mon, 19 May 2025 08:43:07 +0200 Subject: [PATCH 5/6] chore: better descriptions --- specs/ingestion/paths/push.yml | 10 +++++++--- specs/ingestion/paths/tasks/v2/pushTask.yml | 9 ++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/specs/ingestion/paths/push.yml b/specs/ingestion/paths/push.yml index e0ddf303f3c..4e65167ae51 100644 --- a/specs/ingestion/paths/push.yml +++ b/specs/ingestion/paths/push.yml @@ -1,6 +1,11 @@ post: - summary: Push a `batch` request payload through the Pipeline - description: Push a `batch` request payload through the Pipeline. You can check the status of your request with the observability endpoints. + summary: Pushes records through the Pipeline, directly to an index + description: > + Pushes records through the Pipeline, directly to an index. You can make the call synchronous by providing the `watch` parameter, + for asynchronous calls, you can use the observability endpoints and/or debugger dashboard to see the status of your task. + + This method is similar to `pushTask`, but requires an `indexName` instead of a `taskID`. If zero or many tasks are found, + an error will be returned. operationId: push x-acl: - addObject @@ -14,7 +19,6 @@ post: - $ref: '../../common/parameters.yml#/IndexName' - $ref: '../common/parameters.yml#/watch' requestBody: - description: Request body of a Search API `batch` request that will be pushed in the Connectors pipeline. content: application/json: schema: diff --git a/specs/ingestion/paths/tasks/v2/pushTask.yml b/specs/ingestion/paths/tasks/v2/pushTask.yml index c01107b17ec..20ad706fb03 100644 --- a/specs/ingestion/paths/tasks/v2/pushTask.yml +++ b/specs/ingestion/paths/tasks/v2/pushTask.yml @@ -1,8 +1,12 @@ post: tags: - tasks - summary: Push a `batch` request payload through the Pipeline - description: Push a `batch` request payload through the Pipeline. You can check the status of task pushes with the observability endpoints. + summary: Pushes records through the Pipeline, directly to an index + description: > + Pushes records through the Pipeline, directly to an index. You can make the call synchronous by providing the `watch` parameter, + for asynchronous calls, you can use the observability endpoints and/or debugger dashboard to see the status of your task. + + This method is similar to `push`, but requires a `taskID` instead of a `indexName`, which is useful when many `destinations` target the same `indexName`. operationId: pushTask x-acl: - addObject @@ -16,7 +20,6 @@ post: - $ref: '../../../common/parameters.yml#/pathTaskID' - $ref: '../../../common/parameters.yml#/watch' requestBody: - description: Request body of a Search API `batch` request that will be pushed in the Connectors pipeline. content: application/json: schema: From d0d3cceaf4c5c05518c8d9c2b770c6f505779bd3 Mon Sep 17 00:00:00 2001 From: shortcuts Date: Mon, 19 May 2025 13:23:46 +0200 Subject: [PATCH 6/6] chore: description --- specs/ingestion/common/schemas/task.yml | 11 ++++++++++- specs/ingestion/paths/push.yml | 4 +++- specs/ingestion/paths/tasks/v2/pushTask.yml | 4 +++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/specs/ingestion/common/schemas/task.yml b/specs/ingestion/common/schemas/task.yml index 158b7f4e8b9..22d1b5b2b40 100644 --- a/specs/ingestion/common/schemas/task.yml +++ b/specs/ingestion/common/schemas/task.yml @@ -612,7 +612,7 @@ PushTaskPayload: type: object properties: action: - $ref: '../../../common/schemas/Batch.yml#/action' + $ref: '#/action' records: type: array items: @@ -627,3 +627,12 @@ PushTaskPayload: required: - action - records + +action: + type: string + enum: + - addObject + - updateObject + - partialUpdateObject + - partialUpdateObjectNoCreate + description: Type of indexing operation. diff --git a/specs/ingestion/paths/push.yml b/specs/ingestion/paths/push.yml index 4e65167ae51..99befc0e2bb 100644 --- a/specs/ingestion/paths/push.yml +++ b/specs/ingestion/paths/push.yml @@ -1,9 +1,11 @@ post: - summary: Pushes records through the Pipeline, directly to an index + summary: Pushes records to be transformed through the Pipeline, directly to an index description: > Pushes records through the Pipeline, directly to an index. You can make the call synchronous by providing the `watch` parameter, for asynchronous calls, you can use the observability endpoints and/or debugger dashboard to see the status of your task. + If you want to leverage the [pre-indexing data transformation](https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/how-to/transform-your-data/), this is the recommended way of ingesting your records. + This method is similar to `pushTask`, but requires an `indexName` instead of a `taskID`. If zero or many tasks are found, an error will be returned. operationId: push diff --git a/specs/ingestion/paths/tasks/v2/pushTask.yml b/specs/ingestion/paths/tasks/v2/pushTask.yml index 20ad706fb03..16dfe0a85cb 100644 --- a/specs/ingestion/paths/tasks/v2/pushTask.yml +++ b/specs/ingestion/paths/tasks/v2/pushTask.yml @@ -1,11 +1,13 @@ post: tags: - tasks - summary: Pushes records through the Pipeline, directly to an index + summary: Pushes records to be transformed through the Pipeline, directly to an index description: > Pushes records through the Pipeline, directly to an index. You can make the call synchronous by providing the `watch` parameter, for asynchronous calls, you can use the observability endpoints and/or debugger dashboard to see the status of your task. + If you want to leverage the [pre-indexing data transformation](https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/how-to/transform-your-data/), this is the recommended way of ingesting your records. + This method is similar to `push`, but requires a `taskID` instead of a `indexName`, which is useful when many `destinations` target the same `indexName`. operationId: pushTask x-acl: