Skip to content

Commit 37f187b

Browse files
shortcutsshortcuts
committed
feat(clients): add replaceAllObjectsWithTransformation
1 parent 2eec2ac commit 37f187b

File tree

7 files changed

+399
-1
lines changed

7 files changed

+399
-1
lines changed

‎scripts/cts/runCts.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,9 @@ export async function runCts(
155155
assertValidTimeouts(languages.length);
156156
assertChunkWrapperValid(languages.length - skip('dart'));
157157
assertValidReplaceAllObjects(languages.length - skip('dart'));
158-
assertValidReplaceAllObjectsWithTransformation(only('javascript'));
158+
assertValidReplaceAllObjectsWithTransformation(
159+
only('javascript') + only('go') + only('python') + only('java') + only('php'),
160+
);
159161
assertValidAccountCopyIndex(only('javascript'));
160162
assertValidReplaceAllObjectsFailed(languages.length - skip('dart'));
161163
assertValidReplaceAllObjectsScopes(languages.length - skip('dart'));

‎scripts/cts/testServer/replaceAllObjectsWithTransformation.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ function addRoutes(app: Express): void {
9292
)?.[1] as string;
9393
expect(raowtState).to.include.keys(lang);
9494
expect(req.body.action === 'addObject').to.equal(true);
95+
expect(req.query.referenceIndexName === `cts_e2e_replace_all_objects_with_transformation_${lang}`).to.equal(true);
9596

9697
raowtState[lang].pushCount += req.body.records.length;
9798

‎specs/search/helpers/replaceAllObjectsWithTransformation.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@ method:
55
- Records
66
x-available-languages:
77
- javascript
8+
- go
9+
- java
10+
- php
11+
- python
812
operationId: replaceAllObjectsWithTransformation
913
summary: Replace all records in an index
1014
description: |

‎templates/go/search_helpers.mustache‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -599,6 +599,100 @@ func (c *APIClient) ChunkedBatch(indexName string, objects []map[string]any, act
599599
return responses, nil
600600
}
601601

602+
/*
603+
ReplaceAllObjectsWithTransformation is similar to the `replaceAllObjects` method but requires a Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/) to be created first, in order to transform records before indexing them to Algolia. The `region` must have been passed to the client instantiation method.
604+
See https://api-clients-automation.netlify.app/docs/add-new-api-client#5-helpers for implementation details.
605+
606+
@param indexName string - the index name to replace objects into.
607+
@param objects []map[string]any - List of objects to replace.
608+
@param opts ...ReplaceAllObjectsOption - Optional parameters for the request.
609+
@return *ReplaceAllObjectsResponse - The response of the replace all objects operation.
610+
@return error - Error if any.
611+
*/
612+
func (c *APIClient) ReplaceAllObjectsWithTransformation(indexName string, objects []map[string]any, opts ...ReplaceAllObjectsOption) (*ReplaceAllObjectsWithTransformationResponse, error) {
613+
if c.ingestionTransporter == nil {
614+
return nil, reportError("`region` must be provided at client instantiation before calling this method.")
615+
}
616+
617+
tmpIndexName := fmt.Sprintf("%s_tmp_%d", indexName, time.Now().UnixNano())
618+
619+
conf := config{
620+
headerParams: map[string]string{},
621+
scopes: []ScopeType{SCOPE_TYPE_SETTINGS, SCOPE_TYPE_RULES, SCOPE_TYPE_SYNONYMS},
622+
}
623+
624+
for _, opt := range opts {
625+
opt.apply(&conf)
626+
}
627+
628+
opts = append(opts, WithWaitForTasks(true))
629+
630+
copyResp, err := c.OperationIndex(c.NewApiOperationIndexRequest(indexName, NewOperationIndexParams(OPERATION_TYPE_COPY, tmpIndexName, WithOperationIndexParamsScope(conf.scopes))), toRequestOptions(opts)...)
631+
if err != nil {
632+
return nil, err
633+
}
634+
635+
watchResp, err := c.ingestionTransporter.ChunkedPush(tmpIndexName, objects, ingestion.Action(ACTION_ADD_OBJECT), &indexName, toIngestionRequestOptions(toRequestOptions(opts))...)
636+
if err != nil {
637+
_, _ = c.DeleteIndex(c.NewApiDeleteIndexRequest(tmpIndexName))
638+
639+
return nil, err //nolint:wrapcheck
640+
}
641+
642+
_, err = c.WaitForTask(tmpIndexName, copyResp.TaskID, replaceAllObjectsToIterableOptions(opts)...)
643+
if err != nil {
644+
_, _ = c.DeleteIndex(c.NewApiDeleteIndexRequest(tmpIndexName))
645+
646+
return nil, err
647+
}
648+
649+
copyResp, err = c.OperationIndex(c.NewApiOperationIndexRequest(indexName, NewOperationIndexParams(OPERATION_TYPE_COPY, tmpIndexName, WithOperationIndexParamsScope(conf.scopes))), toRequestOptions(opts)...)
650+
if err != nil {
651+
_, _ = c.DeleteIndex(c.NewApiDeleteIndexRequest(tmpIndexName))
652+
653+
return nil, err
654+
}
655+
656+
_, err = c.WaitForTask(tmpIndexName, copyResp.TaskID, replaceAllObjectsToIterableOptions(opts)...)
657+
if err != nil {
658+
_, _ = c.DeleteIndex(c.NewApiDeleteIndexRequest(tmpIndexName))
659+
660+
return nil, err
661+
}
662+
663+
moveResp, err := c.OperationIndex(c.NewApiOperationIndexRequest(tmpIndexName, NewOperationIndexParams(OPERATION_TYPE_MOVE, indexName)), toRequestOptions(opts)...)
664+
if err != nil {
665+
_, _ = c.DeleteIndex(c.NewApiDeleteIndexRequest(tmpIndexName))
666+
667+
return nil, err
668+
}
669+
670+
_, err = c.WaitForTask(tmpIndexName, moveResp.TaskID, replaceAllObjectsToIterableOptions(opts)...)
671+
if err != nil {
672+
_, _ = c.DeleteIndex(c.NewApiDeleteIndexRequest(tmpIndexName))
673+
674+
return nil, err
675+
}
676+
677+
var searchWatchResp []WatchResponse
678+
679+
rawResp, err := json.Marshal(watchResp)
680+
if err != nil {
681+
return nil, fmt.Errorf("unable to convert the ingestion WatchResponse to search WatchResponse: %w", err)
682+
}
683+
684+
err = json.Unmarshal(rawResp, &searchWatchResp)
685+
if err != nil {
686+
return nil, fmt.Errorf("unable to convert the ingestion WatchResponse to search WatchResponse: %w", err)
687+
}
688+
689+
return &ReplaceAllObjectsWithTransformationResponse{
690+
CopyOperationResponse: *copyResp,
691+
WatchResponses: searchWatchResp,
692+
MoveOperationResponse: *moveResp,
693+
}, nil
694+
}
695+
602696
/*
603697
ReplaceAllObjects replaces all objects (records) in the given `indexName` with the given `objects`. A temporary index is created during this process in order to backup your data.
604698
See https://api-clients-automation.netlify.app/docs/add-new-api-client#5-helpers for implementation details.

‎templates/java/api_helpers.mustache‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1304,6 +1304,166 @@ public <T> ReplaceAllObjectsResponse replaceAllObjects(
13041304
}
13051305
}
13061306
1307+
/**
1308+
* Helper: Similar to the `saveObjects` method but requires a Push connector
1309+
* (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/)
1310+
* to be created first, in order to transform records before indexing them to Algolia. The
1311+
* `region` must have been passed to the client instantiation method.
1312+
*
1313+
* @param indexName The `indexName` to replace `objects` in.
1314+
* @param objects The array of `objects` to store in the given Algolia `indexName`.
1315+
* @throws AlgoliaRetryException When the retry has failed on all hosts
1316+
* @throws AlgoliaApiException When the API sends an http error code
1317+
* @throws AlgoliaRuntimeException When an error occurred during the serialization
1318+
*/
1319+
public <T> ReplaceAllObjectsWithTransformationResponse replaceAllObjectsWithTransformation(String indexName, Iterable<T> objects) {
1320+
return replaceAllObjectsWithTransformation(indexName, objects, -1);
1321+
}
1322+
1323+
/**
1324+
* Helper: Similar to the `saveObjects` method but requires a Push connector
1325+
* (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/)
1326+
* to be created first, in order to transform records before indexing them to Algolia. The
1327+
* `region` must have been passed to the client instantiation method.
1328+
*
1329+
* @param indexName The `indexName` to replace `objects` in.
1330+
* @param objects The array of `objects` to store in the given Algolia `indexName`.
1331+
* @param batchSize The size of the chunk of `objects`. The number of `batch` calls will be equal
1332+
* to `length(objects) / batchSize`.
1333+
* @throws AlgoliaRetryException When the retry has failed on all hosts
1334+
* @throws AlgoliaApiException When the API sends an http error code
1335+
* @throws AlgoliaRuntimeException When an error occurred during the serialization
1336+
*/
1337+
public <T> ReplaceAllObjectsWithTransformationResponse replaceAllObjectsWithTransformation(String indexName, Iterable<T> objects, int batchSize) {
1338+
return replaceAllObjectsWithTransformation(indexName, objects, batchSize, null, null);
1339+
}
1340+
1341+
/**
1342+
* Helper: Similar to the `saveObjects` method but requires a Push connector
1343+
* (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/)
1344+
* to be created first, in order to transform records before indexing them to Algolia. The
1345+
* `region` must have been passed to the client instantiation method.
1346+
*
1347+
* @param indexName The `indexName` to replace `objects` in.
1348+
* @param objects The array of `objects` to store in the given Algolia `indexName`.
1349+
* @param batchSize The size of the chunk of `objects`. The number of `batch` calls will be equal
1350+
* to `length(objects) / batchSize`.
1351+
* @param scopes The `scopes` to keep from the index. Defaults to ['settings', 'rules',
1352+
* 'synonyms'].
1353+
* @throws AlgoliaRetryException When the retry has failed on all hosts
1354+
* @throws AlgoliaApiException When the API sends an http error code
1355+
* @throws AlgoliaRuntimeException When an error occurred during the serialization
1356+
*/
1357+
public <T> ReplaceAllObjectsWithTransformationResponse replaceAllObjectsWithTransformation(String indexName, Iterable<T> objects, int batchSize, List<ScopeType> scopes) {
1358+
return replaceAllObjectsWithTransformation(indexName, objects, batchSize, scopes, null);
1359+
}
1360+
1361+
/**
1362+
* Helper: Similar to the `saveObjects` method but requires a Push connector
1363+
* (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/)
1364+
* to be created first, in order to transform records before indexing them to Algolia. The
1365+
* `region` must have been passed to the client instantiation method.
1366+
*
1367+
* @param indexName The `indexName` to replace `objects` in.
1368+
* @param objects The array of `objects` to store in the given Algolia `indexName`.
1369+
* @param batchSize The size of the chunk of `objects`. The number of `batch` calls will be equal
1370+
* to `length(objects) / batchSize`.
1371+
* @param scopes The `scopes` to keep from the index. Defaults to ['settings', 'rules',
1372+
* 'synonyms'].
1373+
* @param requestOptions The requestOptions to send along with the query, they will be merged with
1374+
* the transporter requestOptions. (optional)
1375+
* @throws AlgoliaRetryException When the retry has failed on all hosts
1376+
* @throws AlgoliaApiException When the API sends an http error code
1377+
* @throws AlgoliaRuntimeException When an error occurred during the serialization
1378+
*/
1379+
public <T> ReplaceAllObjectsWithTransformationResponse replaceAllObjectsWithTransformation(
1380+
String indexName,
1381+
Iterable<T> objects,
1382+
int batchSize,
1383+
List<ScopeType> scopes,
1384+
RequestOptions requestOptions
1385+
) {
1386+
if (this.ingestionTransporter == null) {
1387+
throw new AlgoliaRuntimeException("`setTransformationRegion` must have been called before calling this method.");
1388+
}
1389+
1390+
Random rnd = new Random();
1391+
String tmpIndexName = indexName + "_tmp_" + rnd.nextInt(100);
1392+
1393+
if (batchSize == -1) {
1394+
batchSize = 1000;
1395+
}
1396+
1397+
if (scopes == null) {
1398+
scopes = new ArrayList<ScopeType>() {
1399+
{
1400+
add(ScopeType.SETTINGS);
1401+
add(ScopeType.RULES);
1402+
add(ScopeType.SYNONYMS);
1403+
}
1404+
};
1405+
}
1406+
1407+
try {
1408+
// Copy settings, synonyms and rules
1409+
UpdatedAtResponse copyOperationResponse = operationIndex(
1410+
indexName,
1411+
new OperationIndexParams().setOperation(OperationType.COPY).setDestination(tmpIndexName).setScope(scopes),
1412+
requestOptions
1413+
);
1414+
1415+
// Save new objects
1416+
List<WatchResponse> watchResponses =
1417+
this.ingestionTransporter.chunkedPush(
1418+
tmpIndexName,
1419+
objects,
1420+
com.algolia.model.ingestion.Action.ADD_OBJECT,
1421+
true,
1422+
batchSize,
1423+
indexName,
1424+
requestOptions
1425+
);
1426+
1427+
waitForTask(tmpIndexName, copyOperationResponse.getTaskID(), requestOptions);
1428+
1429+
copyOperationResponse = operationIndex(
1430+
indexName,
1431+
new OperationIndexParams().setOperation(OperationType.COPY).setDestination(tmpIndexName).setScope(scopes),
1432+
requestOptions
1433+
);
1434+
waitForTask(tmpIndexName, copyOperationResponse.getTaskID(), requestOptions);
1435+
1436+
// Move temporary index to source index
1437+
UpdatedAtResponse moveOperationResponse = operationIndex(
1438+
tmpIndexName,
1439+
new OperationIndexParams().setOperation(OperationType.MOVE).setDestination(indexName),
1440+
requestOptions
1441+
);
1442+
waitForTask(tmpIndexName, moveOperationResponse.getTaskID(), requestOptions);
1443+
1444+
return new ReplaceAllObjectsWithTransformationResponse()
1445+
.setCopyOperationResponse(copyOperationResponse)
1446+
.setWatchResponses(ingestionResponseToSearchResponse(watchResponses))
1447+
.setMoveOperationResponse(moveOperationResponse);
1448+
} catch (Exception e) {
1449+
deleteIndex(tmpIndexName);
1450+
1451+
throw e;
1452+
}
1453+
}
1454+
1455+
private List<com.algolia.model.search.WatchResponse> ingestionResponseToSearchResponse(List<com.algolia.model.ingestion.WatchResponse> responses) {
1456+
try {
1457+
ObjectMapper mapper = new ObjectMapper();
1458+
String json = mapper.writeValueAsString(responses);
1459+
1460+
return mapper.readValue(json, new TypeReference<List<com.algolia.model.search.WatchResponse>>() {});
1461+
} catch (Exception e) {
1462+
throw new AlgoliaRuntimeException("ingestion WatchResponse cannot be converted to a search WatchResponse");
1463+
}
1464+
}
1465+
1466+
13071467
/**
13081468
* Helper: Generates a secured API key based on the given `parent_api_key` and given
13091469
* `restrictions`.

‎templates/php/api.mustache‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -537,6 +537,72 @@ use Algolia\AlgoliaSearch\Exceptions\NotFoundException;
537537
return new SynonymIterator($indexName, $this, $requestOptions);
538538
}
539539

540+
/**
541+
* Helper: Similar to the `replaceAllObjects` method but requires a Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/) to be created first, in order to transform records before indexing them to Algolia. The `region` must have been passed to the client instantiation method.
542+
*
543+
* @param string $indexName The `indexName` to replace `objects` in.
544+
* @param array $objects The array of `objects` to store in the given Algolia `indexName`.
545+
* @param array $batchSize The size of the chunk of `objects`. The number of `batch` calls will be equal to `length(objects) / batchSize`. Defaults to 1000.
546+
* @param array $requestOptions Request options
547+
*/
548+
public function replaceAllObjectsWithTransformation($indexName, $objects, $batchSize = 1000, $scopes = ['settings', 'rules', 'synonyms'], $requestOptions = [])
549+
{
550+
if (null == $this->ingestionTransporter) {
551+
throw new \InvalidArgumentException('`setTransformationRegion` must have been called before calling this method.');
552+
}
553+
554+
$tmpIndexName = $indexName.'_tmp_'.rand(10000000, 99999999);
555+
556+
try {
557+
$copyOperationResponse = $this->operationIndex(
558+
$indexName,
559+
[
560+
'operation' => 'copy',
561+
'destination' => $tmpIndexName,
562+
'scope' => $scopes,
563+
],
564+
$requestOptions
565+
);
566+
567+
$watchResponses = $this->ingestionTransporter->chunkedPush($tmpIndexName, $objects, 'addObject', true, $batchSize, $indexName, $requestOptions);
568+
569+
$this->waitForTask($tmpIndexName, $copyOperationResponse['taskID']);
570+
571+
$copyOperationResponse = $this->operationIndex(
572+
$indexName,
573+
[
574+
'operation' => 'copy',
575+
'destination' => $tmpIndexName,
576+
'scope' => $scopes,
577+
],
578+
$requestOptions
579+
);
580+
581+
$this->waitForTask($tmpIndexName, $copyOperationResponse['taskID']);
582+
583+
$moveOperationResponse = $this->operationIndex(
584+
$tmpIndexName,
585+
[
586+
'operation' => 'move',
587+
'destination' => $indexName,
588+
],
589+
$requestOptions
590+
);
591+
592+
$this->waitForTask($tmpIndexName, $moveOperationResponse['taskID']);
593+
594+
return [
595+
'copyOperationResponse' => $copyOperationResponse,
596+
'watchResponses' => $watchResponses,
597+
'moveOperationResponse' => $moveOperationResponse,
598+
];
599+
} catch (\Throwable $e) {
600+
$this->deleteIndex($tmpIndexName);
601+
602+
throw $e;
603+
}
604+
}
605+
540606
/**
541607
* Helper: Replace all objects in an index using a temporary one.
542608
* See https://api-clients-automation.netlify.app/docs/add-new-api-client#5-helpers for implementation details.

0 commit comments

Comments
 (0)