diff --git a/.codesandbox/ci.json b/.codesandbox/ci.json index 03a57326c..10d725806 100644 --- a/.codesandbox/ci.json +++ b/.codesandbox/ci.json @@ -1,4 +1,6 @@ { + "buildCommand": "build", "packages": ["packages/*"], - "sandboxes": ["github/algolia/create-instantsearch-app/tree/templates/javascript-client"] + "sandboxes": ["vanilla", "github/algolia/instantsearch/tree/3331ed781d3627bc10e73b6eff89b5037a78a0fc/javascript-client"], + "node": "12" } diff --git a/package.json b/package.json index 9506df0f3..499488bb7 100644 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "bundlesize": [ { "path": "packages/algoliasearch/dist/algoliasearch.umd.js", - "maxSize": "8.4KB" + "maxSize": "9.2KB" }, { "path": "packages/algoliasearch/dist/algoliasearch-lite.umd.js", diff --git a/packages/algoliasearch/src/__tests__/default.test.ts b/packages/algoliasearch/src/__tests__/default.test.ts index a321dc477..40b29a658 100644 --- a/packages/algoliasearch/src/__tests__/default.test.ts +++ b/packages/algoliasearch/src/__tests__/default.test.ts @@ -172,4 +172,47 @@ describe('default preset', () => { expect(client).toHaveProperty('destroy'); } }); + + describe('bridge methods', () => { + const clientWithTransformation = algoliasearch('appId', 'apiKey', { + transformation: { region: 'us' }, + }); + const indexWithTransformation = clientWithTransformation.initIndex('foo'); + const index = client.initIndex('foo'); + + test('throws when missing transformation.region', () => { + // @ts-ignore + expect(() => algoliasearch('APP_ID', 'API_KEY', { transformation: {} })).toThrow( + '`region` must be provided when leveraging the transformation pipeline' + ); + }); + + test('throws when wrong transformation.region', () => { + expect(() => + // @ts-ignore + algoliasearch('APP_ID', 'API_KEY', { transformation: { region: 'cn' } }) + ).toThrow('`region` is required and must be one of the following: eu, us}`'); + }); + + test('throws when calling the transformation methods without init parameters', async () => { + await expect( + index.saveObjectsWithTransformation([{ objectID: 'bar', baz: 42 }], { waitForTasks: true }) + ).rejects.toThrow( + '`transformation.region` must be provided at client instantiation before calling this method.' + ); + + await expect( + index.partialUpdateObjectsWithTransformation([{ objectID: 'bar', baz: 42 }], { + waitForTasks: true, + }) + ).rejects.toThrow( + '`transformation.region` must be provided at client instantiation before calling this method.' + ); + }); + + test('exposes the transformation methods at the root of the client', () => { + expect(indexWithTransformation.saveObjectsWithTransformation).not.toBeUndefined(); + expect(indexWithTransformation.partialUpdateObjectsWithTransformation).not.toBeUndefined(); + }); + }); }); diff --git a/packages/algoliasearch/src/builds/browser.ts b/packages/algoliasearch/src/builds/browser.ts index e3fe076dc..45270ac66 100644 --- a/packages/algoliasearch/src/builds/browser.ts +++ b/packages/algoliasearch/src/builds/browser.ts @@ -203,12 +203,24 @@ import { import { createBrowserXhrRequester } from '@algolia/requester-browser-xhr'; import { createUserAgent, Request, RequestOptions } from '@algolia/transporter'; -import { AlgoliaSearchOptions, InitAnalyticsOptions, InitPersonalizationOptions } from '../types'; +import { + createIngestionClient, + partialUpdateObjectsWithTransformation, + saveObjectsWithTransformation, +} from '../ingestion'; +import { + AlgoliaSearchOptions, + IngestionClient, + IngestionMethods, + InitAnalyticsOptions, + InitPersonalizationOptions, + TransformationOptions, +} from '../types'; export default function algoliasearch( appId: string, apiKey: string, - options?: AlgoliaSearchOptions + options?: AlgoliaSearchOptions & TransformationOptions ): SearchClient { const commonOptions = { appId, @@ -244,6 +256,17 @@ export default function algoliasearch( }); }; + /* eslint functional/no-let: "off" */ + let ingestionTransporter: IngestionClient | undefined; + + if (options && options.transformation) { + if (!options.transformation.region) { + throw new Error('`region` must be provided when leveraging the transformation pipeline'); + } + + ingestionTransporter = createIngestionClient({ ...options, ...commonOptions }); + } + return createSearchClient({ ...searchClientOptions, methods: { @@ -285,50 +308,60 @@ export default function algoliasearch( setDictionarySettings, waitAppTask, customRequest, - initIndex: base => (indexName: string): SearchIndex => { - return initIndex(base)(indexName, { - methods: { - batch, - delete: deleteIndex, - findAnswers, - getObject, - getObjects, - saveObject, - saveObjects, - search, - searchForFacetValues, - waitTask, - setSettings, - getSettings, - partialUpdateObject, - partialUpdateObjects, - deleteObject, - deleteObjects, - deleteBy, - clearObjects, - browseObjects, - getObjectPosition, - findObject, - exists, - saveSynonym, - saveSynonyms, - getSynonym, - searchSynonyms, - browseSynonyms, - deleteSynonym, - clearSynonyms, - replaceAllObjects, - replaceAllSynonyms, - searchRules, - getRule, - deleteRule, - saveRule, - saveRules, - replaceAllRules, - browseRules, - clearRules, - }, - }); + initIndex: base => (indexName: string): SearchIndex & IngestionMethods => { + return { + ...initIndex(base)(indexName, { + methods: { + batch, + delete: deleteIndex, + findAnswers, + getObject, + getObjects, + saveObject, + saveObjects, + search, + searchForFacetValues, + waitTask, + setSettings, + getSettings, + partialUpdateObject, + partialUpdateObjects, + deleteObject, + deleteObjects, + deleteBy, + clearObjects, + browseObjects, + getObjectPosition, + findObject, + exists, + saveSynonym, + saveSynonyms, + getSynonym, + searchSynonyms, + browseSynonyms, + deleteSynonym, + clearSynonyms, + replaceAllObjects, + replaceAllSynonyms, + searchRules, + getRule, + deleteRule, + saveRule, + saveRules, + replaceAllRules, + browseRules, + clearRules, + }, + }), + saveObjectsWithTransformation: saveObjectsWithTransformation( + indexName, + ingestionTransporter + ), + partialUpdateObjectsWithTransformation: partialUpdateObjectsWithTransformation( + indexName, + ingestionTransporter + ), + }; }, initAnalytics: () => (clientOptions?: InitAnalyticsOptions): AnalyticsClient => { return createAnalyticsClient({ @@ -546,7 +579,7 @@ export type SearchIndex = BaseSearchIndex & { }; export type SearchClient = BaseSearchClient & { - readonly initIndex: (indexName: string) => SearchIndex; + readonly initIndex: (indexName: string) => SearchIndex & IngestionMethods; readonly search: ( queries: readonly MultipleQueriesQuery[], requestOptions?: RequestOptions & MultipleQueriesOptions diff --git a/packages/algoliasearch/src/builds/node.ts b/packages/algoliasearch/src/builds/node.ts index aafb3b18e..c24525b8d 100644 --- a/packages/algoliasearch/src/builds/node.ts +++ b/packages/algoliasearch/src/builds/node.ts @@ -205,12 +205,24 @@ import { Destroyable } from '@algolia/requester-common'; import { createNodeHttpRequester } from '@algolia/requester-node-http'; import { createUserAgent, Request, RequestOptions } from '@algolia/transporter'; -import { AlgoliaSearchOptions, InitAnalyticsOptions, InitPersonalizationOptions } from '../types'; +import { + createIngestionClient, + partialUpdateObjectsWithTransformation, + saveObjectsWithTransformation, +} from '../ingestion'; +import { + AlgoliaSearchOptions, + IngestionClient, + IngestionMethods, + InitAnalyticsOptions, + InitPersonalizationOptions, + TransformationOptions, +} from '../types'; export default function algoliasearch( appId: string, apiKey: string, - options?: AlgoliaSearchOptions + options?: AlgoliaSearchOptions & TransformationOptions ): SearchClient { const commonOptions = { appId, @@ -244,6 +256,17 @@ export default function algoliasearch( }); }; + /* eslint functional/no-let: "off" */ + let ingestionTransporter: IngestionClient | undefined; + + if (options && options.transformation) { + if (!options.transformation.region) { + throw new Error('`region` must be provided when leveraging the transformation pipeline'); + } + + ingestionTransporter = createIngestionClient({ ...options, ...commonOptions }); + } + return createSearchClient({ ...searchClientOptions, methods: { @@ -288,50 +311,60 @@ export default function algoliasearch( setDictionarySettings, waitAppTask, customRequest, - initIndex: base => (indexName: string): SearchIndex => { - return initIndex(base)(indexName, { - methods: { - batch, - delete: deleteIndex, - findAnswers, - getObject, - getObjects, - saveObject, - saveObjects, - search, - searchForFacetValues, - waitTask, - setSettings, - getSettings, - partialUpdateObject, - partialUpdateObjects, - deleteObject, - deleteObjects, - deleteBy, - clearObjects, - browseObjects, - getObjectPosition, - findObject, - exists, - saveSynonym, - saveSynonyms, - getSynonym, - searchSynonyms, - browseSynonyms, - deleteSynonym, - clearSynonyms, - replaceAllObjects, - replaceAllSynonyms, - searchRules, - getRule, - deleteRule, - saveRule, - saveRules, - replaceAllRules, - browseRules, - clearRules, - }, - }); + initIndex: base => (indexName: string): SearchIndex & IngestionMethods => { + return { + ...initIndex(base)(indexName, { + methods: { + batch, + delete: deleteIndex, + findAnswers, + getObject, + getObjects, + saveObject, + saveObjects, + search, + searchForFacetValues, + waitTask, + setSettings, + getSettings, + partialUpdateObject, + partialUpdateObjects, + deleteObject, + deleteObjects, + deleteBy, + clearObjects, + browseObjects, + getObjectPosition, + findObject, + exists, + saveSynonym, + saveSynonyms, + getSynonym, + searchSynonyms, + browseSynonyms, + deleteSynonym, + clearSynonyms, + replaceAllObjects, + replaceAllSynonyms, + searchRules, + getRule, + deleteRule, + saveRule, + saveRules, + replaceAllRules, + browseRules, + clearRules, + }, + }), + saveObjectsWithTransformation: saveObjectsWithTransformation( + indexName, + ingestionTransporter + ), + partialUpdateObjectsWithTransformation: partialUpdateObjectsWithTransformation( + indexName, + ingestionTransporter + ), + }; }, initAnalytics: () => (clientOptions?: InitAnalyticsOptions): AnalyticsClient => { return createAnalyticsClient({ @@ -549,7 +582,7 @@ export type SearchIndex = BaseSearchIndex & { }; export type SearchClient = BaseSearchClient & { - readonly initIndex: (indexName: string) => SearchIndex; + readonly initIndex: (indexName: string) => SearchIndex & IngestionMethods; readonly search: ( queries: readonly MultipleQueriesQuery[], requestOptions?: RequestOptions & MultipleQueriesOptions diff --git a/packages/algoliasearch/src/ingestion.ts b/packages/algoliasearch/src/ingestion.ts new file mode 100644 index 000000000..abeb5c311 --- /dev/null +++ b/packages/algoliasearch/src/ingestion.ts @@ -0,0 +1,168 @@ +import { AuthMode, ClientTransporterOptions, createAuth, encode } from '@algolia/client-common'; +import { + BatchActionEnum, + ChunkOptions, + PartialUpdateObjectsOptions, + SaveObjectsOptions, + SearchClientOptions, +} from '@algolia/client-search'; +import { MethodEnum } from '@algolia/requester-common'; +import { CallEnum, createTransporter, RequestOptions } from '@algolia/transporter'; + +import { + IngestionClient, + PushOptions, + PushProps, + TransformationOptions, + WatchResponse, +} from './types'; + +export function createIngestionClient( + options: SearchClientOptions & ClientTransporterOptions & TransformationOptions +): IngestionClient { + if (!options || !options.transformation || !options.transformation.region) { + throw new Error('`region` must be provided when leveraging the transformation pipeline'); + } + + if (options.transformation.region !== 'eu' && options.transformation.region !== 'us') { + throw new Error('`region` is required and must be one of the following: eu, us'); + } + + const appId = options.appId; + + const auth = createAuth(AuthMode.WithinHeaders, appId, options.apiKey); + + const transporter = createTransporter({ + hosts: [ + { + url: `data.${options.transformation.region}.algolia.com`, + accept: CallEnum.ReadWrite, + protocol: 'https', + }, + ], + ...options, + headers: { + ...auth.headers(), + ...{ 'content-type': 'text/plain' }, + ...options.headers, + }, + queryParameters: { + ...auth.queryParameters(), + ...options.queryParameters, + }, + }); + + return { + transporter, + appId, + addAlgoliaAgent(segment: string, version?: string): void { + transporter.userAgent.add({ segment, version }); + transporter.userAgent.add({ segment: 'Ingestion', version }); + transporter.userAgent.add({ segment: 'Ingestion via Algoliasearch' }); + }, + clearCache(): Readonly> { + return Promise.all([ + transporter.requestsCache.clear(), + transporter.responsesCache.clear(), + ]).then(() => undefined); + }, + async push( + { indexName, pushTaskPayload, watch }: PushProps, + requestOptions?: RequestOptions + ): Promise { + if (!indexName) { + throw new Error('Parameter `indexName` is required when calling `push`.'); + } + + if (!pushTaskPayload) { + throw new Error('Parameter `pushTaskPayload` is required when calling `push`.'); + } + + if (!pushTaskPayload.action) { + throw new Error('Parameter `pushTaskPayload.action` is required when calling `push`.'); + } + + if (!pushTaskPayload.records) { + throw new Error('Parameter `pushTaskPayload.records` is required when calling `push`.'); + } + + const opts: RequestOptions = requestOptions || { queryParameters: {} }; + + return await transporter.write( + { + method: MethodEnum.Post, + path: encode('1/push/%s', indexName), + data: pushTaskPayload, + }, + { + ...opts, + queryParameters: { + ...opts.queryParameters, + watch: watch !== undefined, + }, + } + ); + }, + }; +} + +export function saveObjectsWithTransformation(indexName: string, client?: IngestionClient) { + return async ( + objects: ReadonlyArray>>, + requestOptions?: RequestOptions & ChunkOptions & SaveObjectsOptions & PushOptions + ): Promise => { + if (!client) { + throw new Error( + '`options.transformation.region` must be provided at client instantiation before calling this method.' + ); + } + + const { autoGenerateObjectIDIfNotExist, watch, ...rest } = requestOptions || {}; + + const action = autoGenerateObjectIDIfNotExist + ? BatchActionEnum.AddObject + : BatchActionEnum.UpdateObject; + + /* eslint functional/immutable-data: "off" */ + return await client.push( + { + indexName, + pushTaskPayload: { action, records: objects }, + watch, + }, + rest + ); + }; +} + +export function partialUpdateObjectsWithTransformation( + indexName: string, + client?: IngestionClient +) { + return async ( + objects: ReadonlyArray>>, + requestOptions?: RequestOptions & ChunkOptions & PartialUpdateObjectsOptions & PushOptions + ): Promise => { + if (!client) { + throw new Error( + '`options.transformation.region` must be provided at client instantiation before calling this method.' + ); + } + + const { createIfNotExists, watch, ...rest } = requestOptions || {}; + + const action = createIfNotExists + ? BatchActionEnum.PartialUpdateObject + : BatchActionEnum.PartialUpdateObjectNoCreate; + + /* eslint functional/immutable-data: "off" */ + return await client.push( + { + indexName, + pushTaskPayload: { action, records: objects }, + watch, + }, + rest + ); + }; +} diff --git a/packages/algoliasearch/src/types/Ingestion.ts b/packages/algoliasearch/src/types/Ingestion.ts new file mode 100644 index 000000000..04ad3b99d --- /dev/null +++ b/packages/algoliasearch/src/types/Ingestion.ts @@ -0,0 +1,112 @@ +import { + BatchActionType, + ChunkOptions, + PartialUpdateObjectsOptions, + SaveObjectsOptions, + SearchClient as BaseSearchClient, +} from '@algolia/client-search'; +import { RequestOptions } from '@algolia/transporter'; + +export type TransformationOptions = { + // When provided, a second transporter will be created in order to leverage the `*WithTransformation` methods exposed by the Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/). + readonly transformation?: { + // The region of your Algolia application ID, used to target the correct hosts of the transformation service. + readonly region: 'eu' | 'us'; + }; +}; + +export type IngestionMethods = { + /** + * Helper: Similar to the `saveObjects` method but requires a Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/) to be created first, in order to transform records before indexing them to Algolia. The `region` must've been passed to the client instantiation method. + * + * @param objects - The array of `objects` to store in the given Algolia `indexName`. + * @param requestOptions - The requestOptions to send along with the query, they will be forwarded to the `batch` method and merged with the transporter requestOptions. + */ + readonly saveObjectsWithTransformation: ( + objects: ReadonlyArray>>, + requestOptions?: RequestOptions & ChunkOptions & SaveObjectsOptions & PushOptions + ) => Promise; + + /** + * Helper: Similar to the `partialUpdateObjects` method but requires a Push connector (https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/connectors/push/) to be created first, in order to transform records before indexing them to Algolia. The `region` must've been passed to the client instantiation method. + * + * @param objects - The array of `objects` to update in the given Algolia `indexName`. + * @param requestOptions - The requestOptions to send along with the query, they will be forwarded to the `getTask` method and merged with the transporter requestOptions. + */ + readonly partialUpdateObjectsWithTransformation: ( + objects: ReadonlyArray>>, + requestOptions?: RequestOptions & ChunkOptions & PartialUpdateObjectsOptions & PushOptions + ) => Promise; +}; + +export type WatchResponse = { + /** + * Universally unique identifier (UUID) of a task run. + */ + readonly runID: string; + + /** + * Universally unique identifier (UUID) of an event. + */ + readonly eventID?: string; + + /** + * when used with discovering or validating sources, the sampled data of your source is returned. + */ + readonly data?: ReadonlyArray>; + + /** + * in case of error, observability events will be added to the response, if any. + */ + readonly events?: readonly Event[]; + + /** + * a message describing the outcome of a validate run. + */ + readonly message?: string; + + /** + * Date of creation in RFC 3339 format. + */ + readonly createdAt?: string; +}; + +/** + * Properties for the `push` method. + */ +export type PushProps = { + /** + * Name of the index on which to perform the operation. + */ + readonly indexName: string; + readonly pushTaskPayload: { + readonly action: BatchActionType; + readonly records: Record; + }; + /** + * When provided, the push operation will be synchronous and the API will wait for the ingestion to be finished before responding. + */ + readonly watch?: boolean; +}; + +export type PushOptions = Pick; + +export type IngestionClient = BaseSearchClient & { + /** + * Pushes records through the Pipeline, directly to an index. You can make the call synchronous by providing the `watch` parameter, for asynchronous calls, you can use the observability endpoints and/or debugger dashboard to see the status of your task. If you want to leverage the [pre-indexing data transformation](https://www.algolia.com/doc/guides/sending-and-managing-data/send-and-update-your-data/how-to/transform-your-data/), this is the recommended way of ingesting your records. If zero or many tasks are found, an error will be returned. + * + * Required API Key ACLs: + * - addObject + * - deleteIndex + * - editSettings + * @param push - The push object. + * @param push.indexName - Name of the index on which to perform the operation. + * @param push.pushTaskPayload - The pushTaskPayload object. + * @param push.watch - When provided, the push operation will be synchronous and the API will wait for the ingestion to be finished before responding. + * @param requestOptions - The requestOptions to send along with the query, they will be merged with the transporter requestOptions. + */ + readonly push: ( + { indexName, pushTaskPayload, watch }: PushProps, + requestOptions?: RequestOptions + ) => Promise; +}; diff --git a/packages/algoliasearch/src/types/index.ts b/packages/algoliasearch/src/types/index.ts index 10c49a730..97bc0ef90 100644 --- a/packages/algoliasearch/src/types/index.ts +++ b/packages/algoliasearch/src/types/index.ts @@ -3,3 +3,4 @@ */ export * from './AlgoliaSearchOptions'; +export * from './Ingestion';