From eb1e33ef4039cf7b4e5070bb4038e52b3173b162 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Mon, 16 Sep 2024 10:20:48 +0100 Subject: [PATCH 1/6] Add support for generative-anthropic --- src/collections/config/types/generative.ts | 10 ++++++ src/collections/configure/generative.ts | 20 +++++++++++- src/collections/configure/types/generative.ts | 3 ++ src/collections/configure/unit.test.ts | 31 +++++++++++++++++++ 4 files changed, 63 insertions(+), 1 deletion(-) diff --git a/src/collections/config/types/generative.ts b/src/collections/config/types/generative.ts index 14b5dbff..eec5bada 100644 --- a/src/collections/config/types/generative.ts +++ b/src/collections/config/types/generative.ts @@ -14,6 +14,15 @@ export type GenerativeAWSConfig = { endpoint?: string; }; +export type GenerativeAnthropicConfig = { + maxTokens?: number; + model?: string; + stopSequences?: string[]; + temperature?: number; + topK?: number; + topP?: number; +}; + export type GenerativeAnyscaleConfig = { model?: string; temperature?: number; @@ -83,6 +92,7 @@ export type GenerativeConfigType = G extends 'generative-openai' : Record | undefined; export type GenerativeSearch = + | 'generative-anthropic' | 'generative-anyscale' | 'generative-aws' | 'generative-mistral' diff --git a/src/collections/configure/generative.ts b/src/collections/configure/generative.ts index 3ccb6b05..235a02dc 100644 --- a/src/collections/configure/generative.ts +++ b/src/collections/configure/generative.ts @@ -1,5 +1,6 @@ import { GenerativeAWSConfig, + GenerativeAnthropicConfig, GenerativeAnyscaleConfig, GenerativeAzureOpenAIConfig, GenerativeCohereConfig, @@ -12,6 +13,7 @@ import { } from '../config/types/index.js'; import { GenerativeAWSConfigCreate, + GenerativeAnthropicConfigCreate, GenerativeAnyscaleConfigCreate, GenerativeAzureOpenAIConfigCreate, GenerativeCohereConfigCreate, @@ -23,12 +25,28 @@ import { } from '../index.js'; export default { + /** + * Create a `ModuleConfig<'generative-anthropic', GenerativeAnthropicConfig | undefined>` object for use when performing AI generation using the `generative-anthropic` module. + * + * See the [documentation](https://weaviate.io/developers/weaviate/modules/reader-generator-modules/generative-anthropic) for detailed usage. + * + * @param {GenerativeAnthropicConfigCreate} [config] The configuration for the `generative-anthropic` module. + * @returns {ModuleConfig<'generative-anthropic', GenerativeAnthropicConfig | undefined>} The configuration object. + */ + anthropic( + config?: GenerativeAnthropicConfigCreate + ): ModuleConfig<'generative-anthropic', GenerativeAnthropicConfig | undefined> { + return { + name: 'generative-anthropic', + config, + }; + }, /** * Create a `ModuleConfig<'generative-anyscale', GenerativeAnyscaleConfig | undefined>` object for use when performing AI generation using the `generative-anyscale` module. * * See the [documentation](https://weaviate.io/developers/weaviate/modules/reader-generator-modules/generative-anyscale) for detailed usage. * - * @param {GenerativeAnyscaleConfigCreate} config The configuration for the `generative-aws` module. + * @param {GenerativeAnyscaleConfigCreate} [config] The configuration for the `generative-aws` module. * @returns {ModuleConfig<'generative-anyscale', GenerativeAnyscaleConfig | undefined>} The configuration object. */ anyscale( diff --git a/src/collections/configure/types/generative.ts b/src/collections/configure/types/generative.ts index 52f23aaf..c6d050c8 100644 --- a/src/collections/configure/types/generative.ts +++ b/src/collections/configure/types/generative.ts @@ -1,5 +1,6 @@ import { GenerativeAWSConfig, + GenerativeAnthropicConfig, GenerativeAnyscaleConfig, GenerativeMistralConfig, GenerativeOctoAIConfig, @@ -20,6 +21,8 @@ export type GenerativeOpenAIConfigCreate = GenerativeOpenAIConfigBaseCreate & { model?: string; }; +export type GenerativeAnthropicConfigCreate = GenerativeAnthropicConfig; + export type GenerativeAzureOpenAIConfigCreate = GenerativeOpenAIConfigBaseCreate & { resourceName: string; deploymentId: string; diff --git a/src/collections/configure/unit.test.ts b/src/collections/configure/unit.test.ts index f1ffd18d..b33d32ec 100644 --- a/src/collections/configure/unit.test.ts +++ b/src/collections/configure/unit.test.ts @@ -1,5 +1,6 @@ import { GenerativeAWSConfig, + GenerativeAnthropicConfig, GenerativeAnyscaleConfig, GenerativeAzureOpenAIConfig, GenerativeCohereConfig, @@ -1066,6 +1067,36 @@ describe('Unit testing of the vectorizer factory class', () => { }); describe('Unit testing of the generative factory class', () => { + it('should create the correct GenerativeAnthropicConfig type with required & default values', () => { + const config = configure.generative.anthropic(); + expect(config).toEqual>({ + name: 'generative-anthropic', + config: undefined, + }); + }); + + it('should create the correct GenerativeAnthropicConfig type with all values', () => { + const config = configure.generative.anthropic({ + maxTokens: 100, + model: 'model', + stopSequences: ['stop1', 'stop2'], + temperature: 0.5, + topK: 10, + topP: 0.8, + }); + expect(config).toEqual>({ + name: 'generative-anthropic', + config: { + maxTokens: 100, + model: 'model', + stopSequences: ['stop1', 'stop2'], + temperature: 0.5, + topK: 10, + topP: 0.8, + }, + }); + }); + it('should create the correct GenerativeAnyscaleConfig type with required & default values', () => { const config = configure.generative.anyscale(); expect(config).toEqual>({ From b39f14ff66a31189333400cbebbb2e1f6642e2bc Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Mon, 16 Sep 2024 11:16:43 +0100 Subject: [PATCH 2/6] Add support for `text2vec-mistral` --- src/collections/config/types/vectorizer.ts | 15 ++++++++ src/collections/configure/types/vectorizer.ts | 5 +++ src/collections/configure/unit.test.ts | 37 +++++++++++++++++++ src/collections/configure/vectorizer.ts | 21 +++++++++++ 4 files changed, 78 insertions(+) diff --git a/src/collections/config/types/vectorizer.ts b/src/collections/config/types/vectorizer.ts index 16c17998..7fa91efd 100644 --- a/src/collections/config/types/vectorizer.ts +++ b/src/collections/config/types/vectorizer.ts @@ -24,6 +24,7 @@ export type Vectorizer = | 'text2vec-gpt4all' | 'text2vec-huggingface' | 'text2vec-jina' + | 'text2vec-mistral' | 'text2vec-octoai' | 'text2vec-ollama' | 'text2vec-openai' @@ -255,6 +256,18 @@ export type Text2VecJinaConfig = { vectorizeCollectionName?: boolean; }; +/** + * The configuration for text vectorization using Mistral. + * + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/mistral/embeddings) for detailed usage. + */ +export type Text2VecMistralConfig = { + /** The model to use. */ + model?: 'mistral-embed' | string; + /** Whether to vectorize the collection name. */ + vectorizeCollectionName?: boolean; +}; + /** * The configuration for text vectorization using OctoAI. * @@ -393,6 +406,8 @@ export type VectorizerConfigType = V extends 'img2vec-neural' ? Text2VecHuggingFaceConfig | undefined : V extends 'text2vec-jina' ? Text2VecJinaConfig | undefined + : V extends 'text2vec-mistral' + ? Text2VecMistralConfig | undefined : V extends 'text2vec-octoai' ? Text2VecOctoAIConfig | undefined : V extends 'text2vec-ollama' diff --git a/src/collections/configure/types/vectorizer.ts b/src/collections/configure/types/vectorizer.ts index 6f41ccc5..815d1530 100644 --- a/src/collections/configure/types/vectorizer.ts +++ b/src/collections/configure/types/vectorizer.ts @@ -10,6 +10,7 @@ import { Text2VecGPT4AllConfig, Text2VecHuggingFaceConfig, Text2VecJinaConfig, + Text2VecMistralConfig, Text2VecOctoAIConfig, Text2VecOllamaConfig, Text2VecOpenAIConfig, @@ -145,6 +146,8 @@ export type Text2VecHuggingFaceConfigCreate = Text2VecHuggingFaceConfig; export type Text2VecJinaConfigCreate = Text2VecJinaConfig; +export type Text2VecMistralConfigCreate = Text2VecMistralConfig; + export type Text2VecOctoAIConfigCreate = Text2VecOctoAIConfig; export type Text2VecOllamaConfigCreate = Text2VecOllamaConfig; @@ -179,6 +182,8 @@ export type VectorizerConfigCreateType = V extends 'img2vec-neural' ? Text2VecHuggingFaceConfigCreate | undefined : V extends 'text2vec-jina' ? Text2VecJinaConfigCreate | undefined + : V extends 'text2vec-mistral' + ? Text2VecMistralConfigCreate | undefined : V extends 'text2vec-octoai' ? Text2VecOctoAIConfigCreate | undefined : V extends 'text2vec-ollama' diff --git a/src/collections/configure/unit.test.ts b/src/collections/configure/unit.test.ts index b33d32ec..38578863 100644 --- a/src/collections/configure/unit.test.ts +++ b/src/collections/configure/unit.test.ts @@ -823,6 +823,43 @@ describe('Unit testing of the vectorizer factory class', () => { }); }); + it('should create the correct Text2VecMistralConfig type with defaults', () => { + const config = configure.vectorizer.text2VecMistral(); + expect(config).toEqual>({ + name: undefined, + vectorIndex: { + name: 'hnsw', + config: undefined, + }, + vectorizer: { + name: 'text2vec-mistral', + config: undefined, + }, + }); + }); + + it('should create the correct Text2VecMistralConfig type with all values', () => { + const config = configure.vectorizer.text2VecMistral({ + name: 'test', + model: 'model', + vectorizeCollectionName: true, + }); + expect(config).toEqual>({ + name: 'test', + vectorIndex: { + name: 'hnsw', + config: undefined, + }, + vectorizer: { + name: 'text2vec-mistral', + config: { + model: 'model', + vectorizeCollectionName: true, + }, + }, + }); + }); + it('should create the correct Text2VecOctoAIConfig type with defaults', () => { const config = configure.vectorizer.text2VecOctoAI(); expect(config).toEqual>({ diff --git a/src/collections/configure/vectorizer.ts b/src/collections/configure/vectorizer.ts index 60711d8d..de401005 100644 --- a/src/collections/configure/vectorizer.ts +++ b/src/collections/configure/vectorizer.ts @@ -362,6 +362,27 @@ export const vectorizer = { }, }); }, + /** + * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-mistral'`. + * + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/mistral/embeddings) for detailed usage. + * + * @param {ConfigureTextVectorizerOptions} [opts] The configuration for the `text2vec-mistral` vectorizer. + * @returns {VectorConfigCreate, N, I, 'text2vec-mistral'>} The configuration object. + */ + text2VecMistral: ( + opts?: ConfigureTextVectorizerOptions + ): VectorConfigCreate, N, I, 'text2vec-mistral'> => { + const { name, sourceProperties, vectorIndexConfig, ...config } = opts || {}; + return makeVectorizer(name, { + sourceProperties, + vectorIndexConfig, + vectorizerConfig: { + name: 'text2vec-mistral', + config: Object.keys(config).length === 0 ? undefined : config, + }, + }); + }, /** * */ From 604f9dc9bc73c6d44c1885c8b1998d2480efbcd6 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Mon, 16 Sep 2024 11:26:24 +0100 Subject: [PATCH 3/6] Fix module docstrings pointing docs links to new pages --- src/collections/configure/generative.ts | 20 ++++++------- src/collections/configure/reranker.ts | 8 ++--- src/collections/configure/vectorizer.ts | 39 ++++++++++++++----------- 3 files changed, 36 insertions(+), 31 deletions(-) diff --git a/src/collections/configure/generative.ts b/src/collections/configure/generative.ts index 235a02dc..379060c4 100644 --- a/src/collections/configure/generative.ts +++ b/src/collections/configure/generative.ts @@ -28,7 +28,7 @@ export default { /** * Create a `ModuleConfig<'generative-anthropic', GenerativeAnthropicConfig | undefined>` object for use when performing AI generation using the `generative-anthropic` module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/reader-generator-modules/generative-anthropic) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/anthropic/generative) for detailed usage. * * @param {GenerativeAnthropicConfigCreate} [config] The configuration for the `generative-anthropic` module. * @returns {ModuleConfig<'generative-anthropic', GenerativeAnthropicConfig | undefined>} The configuration object. @@ -44,7 +44,7 @@ export default { /** * Create a `ModuleConfig<'generative-anyscale', GenerativeAnyscaleConfig | undefined>` object for use when performing AI generation using the `generative-anyscale` module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/reader-generator-modules/generative-anyscale) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/anyscale/generative) for detailed usage. * * @param {GenerativeAnyscaleConfigCreate} [config] The configuration for the `generative-aws` module. * @returns {ModuleConfig<'generative-anyscale', GenerativeAnyscaleConfig | undefined>} The configuration object. @@ -60,7 +60,7 @@ export default { /** * Create a `ModuleConfig<'generative-aws', GenerativeAWSConfig>` object for use when performing AI generation using the `generative-aws` module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/reader-generator-modules/generative-aws) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/aws/generative) for detailed usage. * * @param {GenerativeAWSConfigCreate} config The configuration for the `generative-aws` module. * @returns {ModuleConfig<'generative-aws', GenerativeAWSConfig>} The configuration object. @@ -74,7 +74,7 @@ export default { /** * Create a `ModuleConfig<'generative-openai', GenerativeAzureOpenAIConfig>` object for use when performing AI generation using the `generative-openai` module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/reader-generator-modules/generative-openai) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/openai/generative) for detailed usage. * * @param {GenerativeAzureOpenAIConfigCreate} config The configuration for the `generative-openai` module. * @returns {ModuleConfig<'generative-openai', GenerativeAzureOpenAIConfig>} The configuration object. @@ -99,7 +99,7 @@ export default { /** * Create a `ModuleConfig<'generative-cohere', GenerativeCohereConfig>` object for use when performing AI generation using the `generative-cohere` module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/reader-generator-modules/generative-cohere) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/cohere/generative) for detailed usage. * * @param {GenerativeCohereConfigCreate} [config] The configuration for the `generative-cohere` module. * @returns {ModuleConfig<'generative-cohere', GenerativeCohereConfig>} The configuration object. @@ -124,7 +124,7 @@ export default { /** * Create a `ModuleConfig<'generative-mistral', GenerativeMistralConfig | undefined>` object for use when performing AI generation using the `generative-mistral` module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/reader-generator-modules/generative-mistral) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/mistral/generative) for detailed usage. * * @param {GenerativeMistralConfigCreate} [config] The configuration for the `generative-mistral` module. * @returns {ModuleConfig<'generative-mistral', GenerativeMistralConfig | undefined>} The configuration object. @@ -140,7 +140,7 @@ export default { /** * Create a `ModuleConfig<'generative-octoai', GenerativeOpenAIConfig | undefined>` object for use when performing AI generation using the `generative-octoai` module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/reader-generator-modules/generative-octoai) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/octoai/generative) for detailed usage. * * @param {GenerativeOctoAIConfigCreate} [config] The configuration for the `generative-octoai` module. * @returns {ModuleConfig<'generative-octoai', GenerativeOctoAIConfig | undefined>} The configuration object. @@ -156,7 +156,7 @@ export default { /** * Create a `ModuleConfig<'generative-ollama', GenerativeOllamaConfig | undefined>` object for use when performing AI generation using the `generative-ollama` module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/reader-generator-modules/generative-ollama) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/ollama/generative) for detailed usage. * * @param {GenerativeOllamaConfigCreate} [config] The configuration for the `generative-openai` module. * @returns {ModuleConfig<'generative-ollama', GenerativeOllamaConfig | undefined>} The configuration object. @@ -172,7 +172,7 @@ export default { /** * Create a `ModuleConfig<'generative-openai', GenerativeOpenAIConfig | undefined>` object for use when performing AI generation using the `generative-openai` module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/reader-generator-modules/generative-openai) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/openai/generative) for detailed usage. * * @param {GenerativeOpenAIConfigCreate} [config] The configuration for the `generative-openai` module. * @returns {ModuleConfig<'generative-openai', GenerativeOpenAIConfig | undefined>} The configuration object. @@ -198,7 +198,7 @@ export default { /** * Create a `ModuleConfig<'generative-palm', GenerativePaLMConfig>` object for use when performing AI generation using the `generative-palm` module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/reader-generator-modules/generative-palm) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/google/generative) for detailed usage. * * @param {GenerativePaLMConfigCreate} [config] The configuration for the `generative-palm` module. * @returns {ModuleConfig<'generative-palm', GenerativePaLMConfig>} The configuration object. diff --git a/src/collections/configure/reranker.ts b/src/collections/configure/reranker.ts index f16a8c00..0070e3ed 100644 --- a/src/collections/configure/reranker.ts +++ b/src/collections/configure/reranker.ts @@ -9,7 +9,7 @@ export default { /** * Create a `ModuleConfig<'reranker-cohere', RerankerCohereConfig>` object for use when reranking using the `reranker-cohere` module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/reranker-cohere) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/cohere/reranker) for detailed usage. * * @param {RerankerCohereConfig} [config] The configuration for the `reranker-cohere` module. * @returns {ModuleConfig<'reranker-cohere', RerankerCohereConfig>} The configuration object. @@ -25,7 +25,7 @@ export default { /** * Create a `ModuleConfig<'reranker-jinaai', RerankerJinaAIConfig>` object for use when reranking using the `reranker-jinaai` module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/reranker-jinaai) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/jinaai/reranker) for detailed usage. * * @param {RerankerJinaAIConfig} [config] The configuration for the `reranker-jinaai` module. * @returns {ModuleConfig<'reranker-jinaai', RerankerJinaAIConfig | undefined>} The configuration object. @@ -41,7 +41,7 @@ export default { /** * Create a `ModuleConfig<'reranker-transformers', Record>` object for use when reranking using the `reranker-transformers` module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/reranker-transformers) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/transformers/reranker) for detailed usage. * * @returns {ModuleConfig<'reranker-transformers', Record>} The configuration object. */ @@ -54,7 +54,7 @@ export default { /** * Create a `ModuleConfig<'reranker-voyageai', RerankerVoyageAIConfig>` object for use when reranking using the `reranker-voyageai` module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/reranker-voyageai) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/voyageai/reranker) for detailed usage. * * @param {RerankerVoyageAIConfig} [config] The configuration for the `reranker-voyage-ai` module. * @returns {ModuleConfig<'reranker-voyage-ai', RerankerVoyageAIConfig | undefined>} The configuration object. diff --git a/src/collections/configure/vectorizer.ts b/src/collections/configure/vectorizer.ts index de401005..de9bb7b4 100644 --- a/src/collections/configure/vectorizer.ts +++ b/src/collections/configure/vectorizer.ts @@ -64,7 +64,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'img2vec-neural'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/img2vec-neural) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/modules/img2vec-neural) for detailed usage. * * @param {ConfigureNonTextVectorizerOptions} [opts] The configuration options for the `img2vec-neural` vectorizer. * @returns {VectorConfigCreate[], N, I, 'img2vec-neural'>} The configuration object. @@ -84,7 +84,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'multi2vec-bind'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/multi2vec-bind) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/imagebind/embeddings-multimodal) for detailed usage. * * @param {ConfigureNonTextVectorizerOptions} [opts] The configuration options for the `multi2vec-bind` vectorizer. * @returns {VectorConfigCreate[], N, I, 'multi2vec-bind'>} The configuration object. @@ -132,7 +132,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'multi2vec-clip'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/multi2vec-clip) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/transformers/embeddings-multimodal) for detailed usage. * * @param {ConfigureNonTextVectorizerOptions} [opts] The configuration options for the `multi2vec-clip` vectorizer. * @returns {VectorConfigCreate[], N, I, 'multi2vec-clip'>} The configuration object. @@ -165,7 +165,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'multi2vec-palm'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/multi2vec-palm) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/google/embeddings-multimodal) for detailed usage. * * @param {ConfigureNonTextVectorizerOptions} opts The configuration options for the `multi2vec-palm` vectorizer. * @returns {VectorConfigCreate[], N, I, 'multi2vec-palm'>} The configuration object. @@ -198,7 +198,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'ref2vec-centroid'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/ref2vec-centroid) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/modules/ref2vec-centroid) for detailed usage. * * @param {ConfigureNonTextVectorizerOptions} opts The configuration options for the `ref2vec-centroid` vectorizer. * @returns {VectorConfigCreate} The configuration object. @@ -218,7 +218,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-aws'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-aws) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/aws/embeddings) for detailed usage. * * @param {ConfigureTextVectorizerOptions} opts The configuration options for the `text2vec-aws` vectorizer. * @returns { VectorConfigCreate, N, I, 'text2vec-aws'>} The configuration object. @@ -239,7 +239,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-azure-openai'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-openai) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/openai/embeddings) for detailed usage. * * @param {ConfigureTextVectorizerOptions} opts The configuration options for the `text2vec-azure-openai` vectorizer. * @returns {VectorConfigCreate, N, I, 'text2vec-azure-openai'>} The configuration object. @@ -260,7 +260,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-cohere'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-cohere) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/cohere/embeddings) for detailed usage. * * @param {ConfigureTextVectorizerOptions} [opts] The configuration options for the `text2vec-cohere` vectorizer. * @returns {VectorConfigCreate, N, I, 'text2vec-cohere'>} The configuration object. @@ -281,7 +281,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-contextionary'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-contextionary) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/modules/text2vec-contextionary) for detailed usage. * * @param {ConfigureTextVectorizerOptions} [opts] The configuration for the `text2vec-contextionary` vectorizer. * @returns {VectorConfigCreate, N, I, 'text2vec-contextionary'>} The configuration object. @@ -302,7 +302,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-gpt4all'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/gpt4all/embeddings) for detailed usage. * * @param {ConfigureTextVectorizerOptions} [opts] The configuration for the `text2vec-contextionary` vectorizer. * @returns {VectorConfigCreate, N, I, 'text2vec-gpt4all'>} The configuration object. @@ -323,7 +323,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-huggingface'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-huggingface) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/huggingface/embeddings) for detailed usage. * * @param {ConfigureTextVectorizerOptions} [opts] The configuration for the `text2vec-contextionary` vectorizer. * @returns {VectorConfigCreate, N, I, 'text2vec-huggingface'>} The configuration object. @@ -344,7 +344,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-jina'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-jina) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/jinaai/embeddings) for detailed usage. * * @param {ConfigureTextVectorizerOptions} [opts] The configuration for the `text2vec-jina` vectorizer. * @returns {VectorConfigCreate, N, I, 'text2vec-jina'>} The configuration object. @@ -384,7 +384,12 @@ export const vectorizer = { }); }, /** + * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-octoai'`. * + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/octoai/embeddings) for detailed usage. + * + * @param {ConfigureTextVectorizerOptions} [opts] The configuration for the `text2vec-octoai` vectorizer. + * @returns {VectorConfigCreate, N, I, 'text2vec-octoai'>} The configuration object. */ text2VecOctoAI: ( opts?: ConfigureTextVectorizerOptions @@ -402,7 +407,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-openai'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-openai) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/openai/embeddings) for detailed usage. * * @param {ConfigureTextVectorizerOptions} [opts] The configuration for the `text2vec-openai` vectorizer. * @returns {VectorConfigCreate, N, I, 'text2vec-openai'>} The configuration object. @@ -423,7 +428,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-ollama'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-ollama) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/ollama/embeddings) for detailed usage. * * @param {ConfigureTextVectorizerOptions} [opts] The configuration for the `text2vec-ollama` vectorizer. * @returns {VectorConfigCreate, N, I, 'text2vec-ollama'>} The configuration object. @@ -444,7 +449,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-palm'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-palm) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/google/embeddings) for detailed usage. * * @param {ConfigureTextVectorizerOptions} opts The configuration for the `text2vec-palm` vectorizer. * @returns {VectorConfigCreate, N, I, 'text2vec-palm'>} The configuration object. @@ -465,7 +470,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-transformers'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-transformers) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/transformers/embeddings) for detailed usage. * * @param {ConfigureTextVectorizerOptions} [opts] The configuration for the `text2vec-transformers` vectorizer. * @returns {VectorConfigCreate, N, I, 'text2vec-transformers'>} The configuration object. @@ -486,7 +491,7 @@ export const vectorizer = { /** * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-voyageai'`. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-voyageai) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/voyageai/embeddings) for detailed usage. * * @param {ConfigureTextVectorizerOptions} [opts] The configuration for the `text2vec-voyageai` vectorizer. * @returns {VectorConfigCreate, N, I, 'text2vec-voyageai'>} The configuration object. From b450b77aa625bd50e2e575d6c127427778769759 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Mon, 16 Sep 2024 11:58:45 +0100 Subject: [PATCH 4/6] Add support for `generative-friendliai` --- src/collections/config/types/generative.ts | 33 +++++++++++++- src/collections/config/types/reranker.ts | 1 + src/collections/configure/generative.ts | 15 +++++++ src/collections/configure/types/generative.ts | 45 ++++++++++++++----- src/collections/configure/unit.test.ts | 27 +++++++++++ src/collections/index.ts | 4 +- 6 files changed, 112 insertions(+), 13 deletions(-) diff --git a/src/collections/config/types/generative.ts b/src/collections/config/types/generative.ts index eec5bada..a8f2df18 100644 --- a/src/collections/config/types/generative.ts +++ b/src/collections/config/types/generative.ts @@ -28,6 +28,13 @@ export type GenerativeAnyscaleConfig = { temperature?: number; }; +export type GenerativeFriendliAIConfig = { + baseURL?: string; + maxTokens?: number; + model?: string; + temperature?: number; +}; + export type GenerativeMistralConfig = { maxTokens?: number; model?: string; @@ -75,15 +82,36 @@ export type GenerativePaLMConfig = { }; export type GenerativeConfig = - | GenerativeOpenAIConfig + | GenerativeAnthropicConfig + | GenerativeAnyscaleConfig + | GenerativeAWSConfig + | GenerativeAzureOpenAIConfig | GenerativeCohereConfig + | GenerativeMistralConfig + | GenerativeOctoAIConfig + | GenerativeOllamaConfig + | GenerativeOpenAIConfig | GenerativePaLMConfig | Record | undefined; -export type GenerativeConfigType = G extends 'generative-openai' +export type GenerativeConfigType = G extends 'generative-anthropic' + ? GenerativeAnthropicConfig + : G extends 'generative-anyscale' + ? GenerativeAnyscaleConfig + : G extends 'generative-aws' + ? GenerativeAWSConfig + : G extends 'generative-azure-openai' ? GenerativeOpenAIConfig : G extends 'generative-cohere' + ? GenerativeAzureOpenAIConfig + : G extends 'generative-mistral' + ? GenerativeMistralConfig + : G extends 'generative-octoai' + ? GenerativeOctoAIConfig + : G extends 'generative-ollama' + ? GenerativeOllamaConfig + : G extends 'generative-openai' ? GenerativeCohereConfig : G extends 'generative-palm' ? GenerativePaLMConfig @@ -95,6 +123,7 @@ export type GenerativeSearch = | 'generative-anthropic' | 'generative-anyscale' | 'generative-aws' + | 'generative-azure-openai' | 'generative-mistral' | 'generative-octoai' | 'generative-ollama' diff --git a/src/collections/config/types/reranker.ts b/src/collections/config/types/reranker.ts index 48d1e7d1..f1cd64de 100644 --- a/src/collections/config/types/reranker.ts +++ b/src/collections/config/types/reranker.ts @@ -20,6 +20,7 @@ export type RerankerJinaAIConfig = { export type RerankerConfig = | RerankerCohereConfig + | RerankerJinaAIConfig | RerankerTransformersConfig | RerankerVoyageAIConfig | Record diff --git a/src/collections/configure/generative.ts b/src/collections/configure/generative.ts index 379060c4..7d88e9c3 100644 --- a/src/collections/configure/generative.ts +++ b/src/collections/configure/generative.ts @@ -4,6 +4,7 @@ import { GenerativeAnyscaleConfig, GenerativeAzureOpenAIConfig, GenerativeCohereConfig, + GenerativeFriendliAIConfig, GenerativeMistralConfig, GenerativeOctoAIConfig, GenerativeOllamaConfig, @@ -17,6 +18,7 @@ import { GenerativeAnyscaleConfigCreate, GenerativeAzureOpenAIConfigCreate, GenerativeCohereConfigCreate, + GenerativeFriendliAIConfigCreate, GenerativeMistralConfigCreate, GenerativeOctoAIConfigCreate, GenerativeOllamaConfigCreate, @@ -121,6 +123,19 @@ export default { : undefined, }; }, + /** + * Create a `ModuleConfig<'generative-friendliai', GenerativeFriendliAIConfig | undefined>` object for use when performing AI generation using the `generative-friendliai` module. + * + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/friendliai/generative) for detailed usage. + */ + friendliai( + config?: GenerativeFriendliAIConfigCreate + ): ModuleConfig<'generative-friendliai', GenerativeFriendliAIConfig | undefined> { + return { + name: 'generative-friendliai', + config, + }; + }, /** * Create a `ModuleConfig<'generative-mistral', GenerativeMistralConfig | undefined>` object for use when performing AI generation using the `generative-mistral` module. * diff --git a/src/collections/configure/types/generative.ts b/src/collections/configure/types/generative.ts index c6d050c8..74acacf4 100644 --- a/src/collections/configure/types/generative.ts +++ b/src/collections/configure/types/generative.ts @@ -2,6 +2,7 @@ import { GenerativeAWSConfig, GenerativeAnthropicConfig, GenerativeAnyscaleConfig, + GenerativeFriendliAIConfig, GenerativeMistralConfig, GenerativeOctoAIConfig, GenerativeOllamaConfig, @@ -17,12 +18,12 @@ export type GenerativeOpenAIConfigBaseCreate = { topP?: number; }; -export type GenerativeOpenAIConfigCreate = GenerativeOpenAIConfigBaseCreate & { - model?: string; -}; - export type GenerativeAnthropicConfigCreate = GenerativeAnthropicConfig; +export type GenerativeAnyscaleConfigCreate = GenerativeAnyscaleConfig; + +export type GenerativeAWSConfigCreate = GenerativeAWSConfig; + export type GenerativeAzureOpenAIConfigCreate = GenerativeOpenAIConfigBaseCreate & { resourceName: string; deploymentId: string; @@ -37,9 +38,7 @@ export type GenerativeCohereConfigCreate = { temperature?: number; }; -export type GenerativeAnyscaleConfigCreate = GenerativeAnyscaleConfig; - -export type GenerativeAWSConfigCreate = GenerativeAWSConfig; +export type GenerativeFriendliAIConfigCreate = GenerativeFriendliAIConfig; export type GenerativeMistralConfigCreate = GenerativeMistralConfig; @@ -47,19 +46,45 @@ export type GenerativeOctoAIConfigCreate = GenerativeOctoAIConfig; export type GenerativeOllamaConfigCreate = GenerativeOllamaConfig; +export type GenerativeOpenAIConfigCreate = GenerativeOpenAIConfigBaseCreate & { + model?: string; +}; + export type GenerativePaLMConfigCreate = GenerativePaLMConfig; export type GenerativeConfigCreate = - | GenerativeOpenAIConfigCreate + | GenerativeAnthropicConfigCreate + | GenerativeAnyscaleConfigCreate + | GenerativeAWSConfigCreate + | GenerativeAzureOpenAIConfigCreate | GenerativeCohereConfigCreate + | GenerativeFriendliAIConfigCreate + | GenerativeMistralConfigCreate + | GenerativeOctoAIConfigCreate + | GenerativeOllamaConfigCreate + | GenerativeOpenAIConfigCreate | GenerativePaLMConfigCreate | Record | undefined; -export type GenerativeConfigCreateType = G extends 'generative-openai' - ? GenerativeOpenAIConfigCreate +export type GenerativeConfigCreateType = G extends 'generative-anthropic' + ? GenerativeAnthropicConfigCreate + : G extends 'generative-aws' + ? GenerativeAWSConfigCreate + : G extends 'generative-azure-openai' + ? GenerativeAzureOpenAIConfigCreate : G extends 'generative-cohere' ? GenerativeCohereConfigCreate + : G extends 'generative-friendliai' + ? GenerativeFriendliAIConfigCreate + : G extends 'generative-mistral' + ? GenerativeMistralConfigCreate + : G extends 'generative-octoai' + ? GenerativeOctoAIConfigCreate + : G extends 'generative-ollama' + ? GenerativeOllamaConfigCreate + : G extends 'generative-openai' + ? GenerativeOpenAIConfigCreate : G extends 'generative-palm' ? GenerativePaLMConfigCreate : G extends 'none' diff --git a/src/collections/configure/unit.test.ts b/src/collections/configure/unit.test.ts index 38578863..49d879e6 100644 --- a/src/collections/configure/unit.test.ts +++ b/src/collections/configure/unit.test.ts @@ -4,6 +4,7 @@ import { GenerativeAnyscaleConfig, GenerativeAzureOpenAIConfig, GenerativeCohereConfig, + GenerativeFriendliAIConfig, GenerativeMistralConfig, GenerativeOctoAIConfig, GenerativeOllamaConfig, @@ -1258,6 +1259,32 @@ describe('Unit testing of the generative factory class', () => { }); }); + it('should create the correct GenerativeFriendliAIConfig type with required & default values', () => { + const config = configure.generative.friendliai(); + expect(config).toEqual>({ + name: 'generative-friendliai', + config: undefined, + }); + }); + + it('should create the correct GenerativeFriendliAIConfig type with all values', () => { + const config = configure.generative.friendliai({ + baseURL: 'base-url', + maxTokens: 100, + model: 'model', + temperature: 0.5, + }); + expect(config).toEqual>({ + name: 'generative-friendliai', + config: { + baseURL: 'base-url', + maxTokens: 100, + model: 'model', + temperature: 0.5, + }, + }); + }); + it('should create the correct GenerativeMistralConfig type with required & default values', () => { const config = configure.generative.mistral(); expect(config).toEqual>({ diff --git a/src/collections/index.ts b/src/collections/index.ts index da7d49d5..0472c459 100644 --- a/src/collections/index.ts +++ b/src/collections/index.ts @@ -129,7 +129,9 @@ const collections = (connection: Connection, dbVersionSupport: DbVersionSupport) const moduleConfig: any = {}; if (config.generative) { - moduleConfig[config.generative.name] = config.generative.config ? config.generative.config : {}; + const generative = + config.generative.name === 'generative-azure-openai' ? 'generative-openai' : config.generative.name; + moduleConfig[generative] = config.generative.config ? config.generative.config : {}; } if (config.reranker) { moduleConfig[config.reranker.name] = config.reranker.config ? config.reranker.config : {}; From 8f35761e88b3c41ef1d79141e5bb13fba7d8fbe6 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Mon, 16 Sep 2024 12:20:55 +0100 Subject: [PATCH 5/6] Add support for `generative-databricks` --- src/collections/config/types/generative.ts | 36 ++++++++++++------- src/collections/configure/generative.ts | 20 ++++++++++- src/collections/configure/types/generative.ts | 6 ++++ src/collections/configure/unit.test.ts | 33 +++++++++++++++++ 4 files changed, 82 insertions(+), 13 deletions(-) diff --git a/src/collections/config/types/generative.ts b/src/collections/config/types/generative.ts index a8f2df18..4ad0dc43 100644 --- a/src/collections/config/types/generative.ts +++ b/src/collections/config/types/generative.ts @@ -28,6 +28,23 @@ export type GenerativeAnyscaleConfig = { temperature?: number; }; +export type GenerativeCohereConfig = { + kProperty?: number; + model?: string; + maxTokensProperty?: number; + returnLikelihoodsProperty?: string; + stopSequencesProperty?: string[]; + temperatureProperty?: number; +}; + +export type GenerativeDatabricksConfig = { + endpoint: string; + maxTokens?: number; + temperature?: number; + topK?: number; + topP?: number; +}; + export type GenerativeFriendliAIConfig = { baseURL?: string; maxTokens?: number; @@ -62,15 +79,6 @@ export type GenerativeAzureOpenAIConfig = GenerativeOpenAIConfigBase & { deploymentId: string; }; -export type GenerativeCohereConfig = { - kProperty?: number; - model?: string; - maxTokensProperty?: number; - returnLikelihoodsProperty?: string; - stopSequencesProperty?: string[]; - temperatureProperty?: number; -}; - export type GenerativePaLMConfig = { apiEndpoint?: string; maxOutputTokens?: number; @@ -105,6 +113,10 @@ export type GenerativeConfigType = G extends 'generative-anthropic' ? GenerativeOpenAIConfig : G extends 'generative-cohere' ? GenerativeAzureOpenAIConfig + : G extends 'generative-databricks' + ? GenerativeDatabricksConfig + : G extends 'generative-friendliai' + ? GenerativeFriendliAIConfig : G extends 'generative-mistral' ? GenerativeMistralConfig : G extends 'generative-octoai' @@ -112,8 +124,6 @@ export type GenerativeConfigType = G extends 'generative-anthropic' : G extends 'generative-ollama' ? GenerativeOllamaConfig : G extends 'generative-openai' - ? GenerativeCohereConfig - : G extends 'generative-palm' ? GenerativePaLMConfig : G extends 'none' ? undefined @@ -124,11 +134,13 @@ export type GenerativeSearch = | 'generative-anyscale' | 'generative-aws' | 'generative-azure-openai' + | 'generative-cohere' + | 'generative-databricks' + | 'generative-friendliai' | 'generative-mistral' | 'generative-octoai' | 'generative-ollama' | 'generative-openai' - | 'generative-cohere' | 'generative-palm' | 'none' | string; diff --git a/src/collections/configure/generative.ts b/src/collections/configure/generative.ts index 7d88e9c3..ebe8b15a 100644 --- a/src/collections/configure/generative.ts +++ b/src/collections/configure/generative.ts @@ -4,6 +4,7 @@ import { GenerativeAnyscaleConfig, GenerativeAzureOpenAIConfig, GenerativeCohereConfig, + GenerativeDatabricksConfig, GenerativeFriendliAIConfig, GenerativeMistralConfig, GenerativeOctoAIConfig, @@ -18,6 +19,7 @@ import { GenerativeAnyscaleConfigCreate, GenerativeAzureOpenAIConfigCreate, GenerativeCohereConfigCreate, + GenerativeDatabricksConfigCreate, GenerativeFriendliAIConfigCreate, GenerativeMistralConfigCreate, GenerativeOctoAIConfigCreate, @@ -104,7 +106,7 @@ export default { * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/cohere/generative) for detailed usage. * * @param {GenerativeCohereConfigCreate} [config] The configuration for the `generative-cohere` module. - * @returns {ModuleConfig<'generative-cohere', GenerativeCohereConfig>} The configuration object. + * @returns {ModuleConfig<'generative-cohere', GenerativeCohereConfig | undefined>} The configuration object. */ cohere: ( config?: GenerativeCohereConfigCreate @@ -123,6 +125,22 @@ export default { : undefined, }; }, + /** + * Create a `ModuleConfig<'generative-databricks', GenerativeDatabricksConfig>` object for use when performing AI generation using the `generative-databricks` module. + * + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/databricks/generative) for detailed usage. + * + * @param {GenerativeDatabricksConfigCreate} config The configuration for the `generative-databricks` module. + * @returns {ModuleConfig<'generative-databricks', GenerativeDatabricksConfig>} The configuration object. + */ + databricks: ( + config: GenerativeDatabricksConfigCreate + ): ModuleConfig<'generative-databricks', GenerativeDatabricksConfig> => { + return { + name: 'generative-databricks', + config, + }; + }, /** * Create a `ModuleConfig<'generative-friendliai', GenerativeFriendliAIConfig | undefined>` object for use when performing AI generation using the `generative-friendliai` module. * diff --git a/src/collections/configure/types/generative.ts b/src/collections/configure/types/generative.ts index 74acacf4..59df7f88 100644 --- a/src/collections/configure/types/generative.ts +++ b/src/collections/configure/types/generative.ts @@ -2,6 +2,7 @@ import { GenerativeAWSConfig, GenerativeAnthropicConfig, GenerativeAnyscaleConfig, + GenerativeDatabricksConfig, GenerativeFriendliAIConfig, GenerativeMistralConfig, GenerativeOctoAIConfig, @@ -38,6 +39,8 @@ export type GenerativeCohereConfigCreate = { temperature?: number; }; +export type GenerativeDatabricksConfigCreate = GenerativeDatabricksConfig; + export type GenerativeFriendliAIConfigCreate = GenerativeFriendliAIConfig; export type GenerativeMistralConfigCreate = GenerativeMistralConfig; @@ -58,6 +61,7 @@ export type GenerativeConfigCreate = | GenerativeAWSConfigCreate | GenerativeAzureOpenAIConfigCreate | GenerativeCohereConfigCreate + | GenerativeDatabricksConfigCreate | GenerativeFriendliAIConfigCreate | GenerativeMistralConfigCreate | GenerativeOctoAIConfigCreate @@ -75,6 +79,8 @@ export type GenerativeConfigCreateType = G extends 'generative-anthropic' ? GenerativeAzureOpenAIConfigCreate : G extends 'generative-cohere' ? GenerativeCohereConfigCreate + : G extends 'generative-databricks' + ? GenerativeDatabricksConfigCreate : G extends 'generative-friendliai' ? GenerativeFriendliAIConfigCreate : G extends 'generative-mistral' diff --git a/src/collections/configure/unit.test.ts b/src/collections/configure/unit.test.ts index 49d879e6..9bcadd80 100644 --- a/src/collections/configure/unit.test.ts +++ b/src/collections/configure/unit.test.ts @@ -4,6 +4,7 @@ import { GenerativeAnyscaleConfig, GenerativeAzureOpenAIConfig, GenerativeCohereConfig, + GenerativeDatabricksConfig, GenerativeFriendliAIConfig, GenerativeMistralConfig, GenerativeOctoAIConfig, @@ -1259,6 +1260,38 @@ describe('Unit testing of the generative factory class', () => { }); }); + it('should create the correct GenerativeDatabricksConfig type with required & default values', () => { + const config = configure.generative.databricks({ + endpoint: 'endpoint', + }); + expect(config).toEqual>({ + name: 'generative-databricks', + config: { + endpoint: 'endpoint', + }, + }); + }); + + it('should create the correct GenerativeDatabricksConfig type with all values', () => { + const config = configure.generative.databricks({ + endpoint: 'endpoint', + maxTokens: 100, + temperature: 0.5, + topK: 10, + topP: 0.8, + }); + expect(config).toEqual>({ + name: 'generative-databricks', + config: { + endpoint: 'endpoint', + maxTokens: 100, + temperature: 0.5, + topK: 10, + topP: 0.8, + }, + }); + }); + it('should create the correct GenerativeFriendliAIConfig type with required & default values', () => { const config = configure.generative.friendliai(); expect(config).toEqual>({ From e27fbfc0908f85b4c60f363ebeac83bdc977f874 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Mon, 16 Sep 2024 12:48:21 +0100 Subject: [PATCH 6/6] Add support for `text2vec-databricks` --- src/collections/config/types/vectorizer.ts | 87 +++++++++++-------- src/collections/configure/types/vectorizer.ts | 5 ++ src/collections/configure/unit.test.ts | 44 ++++++++++ src/collections/configure/vectorizer.ts | 21 +++++ 4 files changed, 123 insertions(+), 34 deletions(-) diff --git a/src/collections/config/types/vectorizer.ts b/src/collections/config/types/vectorizer.ts index 7fa91efd..208125fe 100644 --- a/src/collections/config/types/vectorizer.ts +++ b/src/collections/config/types/vectorizer.ts @@ -21,6 +21,7 @@ export type Vectorizer = | 'text2vec-azure-openai' | 'text2vec-cohere' | 'text2vec-contextionary' + | 'text2vec-databricks' | 'text2vec-gpt4all' | 'text2vec-huggingface' | 'text2vec-jina' @@ -33,9 +34,9 @@ export type Vectorizer = | 'text2vec-voyageai' | 'none'; -/** The configuration for image vectorization using a neural network. +/** The configuration for image vectorization using a neural network module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/img2vec-neural) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/modules/img2vec-neural) for detailed usage. */ export type Img2VecNeuralConfig = { /** The image fields used when vectorizing. This is a required field and must match the property fields of the collection that are defined as `DataType.BLOB`. */ @@ -50,9 +51,9 @@ export type Multi2VecField = { weight?: number; }; -/** The configuration for multi-media vectorization using CLIP. +/** The configuration for multi-media vectorization using the CLIP module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/multi2vec-clip) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/transformers/embeddings-multimodal) for detailed usage. */ export type Multi2VecClipConfig = { /** The image fields used when vectorizing. */ @@ -72,9 +73,9 @@ export type Multi2VecClipConfig = { }; }; -/** The configuration for multi-media vectorization using Bind. +/** The configuration for multi-media vectorization using the Bind module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/multi2vec-bind) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/imagebind/embeddings-multimodal) for detailed usage. */ export type Multi2VecBindConfig = { /** The audio fields used when vectorizing. */ @@ -112,9 +113,9 @@ export type Multi2VecBindConfig = { }; }; -/** The configuration for multi-media vectorization using Palm. +/** The configuration for multi-media vectorization using the PaLM model. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-palm) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/google/embeddings) for detailed usage. */ export type Multi2VecPalmConfig = { /** The project ID of the Palm model. */ @@ -144,9 +145,9 @@ export type Multi2VecPalmConfig = { }; }; -/** The configuration for reference-based vectorization using centroids. +/** The configuration for reference-based vectorization using the centroid method. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/ref2vec-centroid) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/modules/ref2vec-centroid) for detailed usage. */ export type Ref2VecCentroidConfig = { /** The properties used as reference points for vectorization. */ @@ -155,7 +156,7 @@ export type Ref2VecCentroidConfig = { method: 'mean' | string; }; -/** The configuration for text vectorization using AWS. +/** The configuration for text vectorization using the AWS module. * * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-aws) for detailed usage. */ @@ -172,9 +173,9 @@ export type Text2VecAWSConfig = { vectorizeCollectionName?: boolean; }; -/** The configuration for text vectorization using Azure OpenAI. +/** The configuration for text vectorization using the OpenAI module with Azure. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-azure-openai) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/openai/embeddings) for detailed usage. */ export type Text2VecAzureOpenAIConfig = { /** The base URL to use where API requests should go. */ @@ -187,9 +188,9 @@ export type Text2VecAzureOpenAIConfig = { vectorizeCollectionName?: boolean; }; -/** The configuration for text vectorization using Cohere. +/** The configuration for text vectorization using the Cohere module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-cohere) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/cohere/embeddings) for detailed usage. */ export type Text2VecCohereConfig = { /** The base URL to use where API requests should go. */ @@ -202,18 +203,28 @@ export type Text2VecCohereConfig = { vectorizeCollectionName?: boolean; }; -/** The configuration for text vectorization using Contextionary. +/** The configuration for text vectorization using the Contextionary module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-contextionary) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/modules/text2vec-contextionary) for detailed usage. */ export type Text2VecContextionaryConfig = { /** Whether to vectorize the collection name. */ vectorizeCollectionName?: boolean; }; -/** The configuration for text vectorization using GPT4All. +/** The configuration for text vectorization using the Databricks module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-gpt4all) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/databricks/embeddings) for detailed usage. + */ +export type Text2VecDatabricksConfig = { + endpoint: string; + instruction?: string; + vectorizeCollectionName?: boolean; +}; + +/** The configuration for text vectorization using the GPT-4-All module. + * + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/gpt4all/embeddings) for detailed usage. */ export type Text2VecGPT4AllConfig = { /** Whether to vectorize the collection name. */ @@ -221,9 +232,9 @@ export type Text2VecGPT4AllConfig = { }; /** - * The configuration for text vectorization using Hugging Face. + * The configuration for text vectorization using the HuggingFace module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-huggingface) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/huggingface/embeddings) for detailed usage. */ export type Text2VecHuggingFaceConfig = { /** The endpoint URL to use. */ @@ -245,9 +256,9 @@ export type Text2VecHuggingFaceConfig = { }; /** - * The configuration for text vectorization using Jina. + * The configuration for text vectorization using the Jina module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-jina) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/jinaai/embeddings) for detailed usage. */ export type Text2VecJinaConfig = { /** The model to use. */ @@ -257,7 +268,7 @@ export type Text2VecJinaConfig = { }; /** - * The configuration for text vectorization using Mistral. + * The configuration for text vectorization using the Mistral module. * * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/mistral/embeddings) for detailed usage. */ @@ -269,9 +280,9 @@ export type Text2VecMistralConfig = { }; /** - * The configuration for text vectorization using OctoAI. + * The configuration for text vectorization using the OctoAI module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-octoai) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/octoai/embeddings) for detailed usage. */ export type Text2VecOctoAIConfig = { /** The base URL to use where API requests should go. */ @@ -283,9 +294,9 @@ export type Text2VecOctoAIConfig = { }; /** - * The configuration for text vectorization using Ollama. + * The configuration for text vectorization using the Ollama module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-ollama) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/ollama/embeddings) for detailed usage. */ export type Text2VecOllamaConfig = { /** The base URL to use where API requests should go. */ @@ -297,9 +308,9 @@ export type Text2VecOllamaConfig = { }; /** - * The configuration for text vectorization using OpenAI. + * The configuration for text vectorization using the OpenAI module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-openai) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/openai/embeddings) for detailed usage. */ export type Text2VecOpenAIConfig = { /** The base URL to use where API requests should go. */ @@ -317,9 +328,9 @@ export type Text2VecOpenAIConfig = { }; /** - * The configuration for text vectorization using Palm. + * The configuration for text vectorization using the PaLM module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-palm) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/google/embeddings) for detailed usage. */ export type Text2VecPalmConfig = { /** The API endpoint to use without a leading scheme such as `http://`. */ @@ -334,6 +345,11 @@ export type Text2VecPalmConfig = { vectorizeCollectionName?: boolean; }; +/** + * The configuration for text vectorization using the Transformers module. + * + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/transformers/embeddings) for detailed usage. + */ export type Text2VecTransformersConfig = { /** The inference url to use where API requests should go. You can use either this OR (`passage_inference_url` & `query_inference_url`). */ inferenceUrl?: string; @@ -348,9 +364,9 @@ export type Text2VecTransformersConfig = { }; /** - * The configuration for text vectorization using Voyage AI. + * The configuration for text vectorization using the VoyageAI module. * - * See the [documentation](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules/text2vec-voyageai) for detailed usage. + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/voyageai/embeddings) for detailed usage. */ export type Text2VecVoyageAIConfig = { /** The base URL to use where API requests should go. */ @@ -375,6 +391,7 @@ export type VectorizerConfig = | Text2VecAzureOpenAIConfig | Text2VecContextionaryConfig | Text2VecCohereConfig + | Text2VecDatabricksConfig | Text2VecGPT4AllConfig | Text2VecHuggingFaceConfig | Text2VecJinaConfig @@ -400,6 +417,8 @@ export type VectorizerConfigType = V extends 'img2vec-neural' ? Text2VecContextionaryConfig | undefined : V extends 'text2vec-cohere' ? Text2VecCohereConfig | undefined + : V extends 'text2vec-databricks' + ? Text2VecDatabricksConfig : V extends 'text2vec-gpt4all' ? Text2VecGPT4AllConfig | undefined : V extends 'text2vec-huggingface' diff --git a/src/collections/configure/types/vectorizer.ts b/src/collections/configure/types/vectorizer.ts index 815d1530..46b88cc3 100644 --- a/src/collections/configure/types/vectorizer.ts +++ b/src/collections/configure/types/vectorizer.ts @@ -7,6 +7,7 @@ import { Text2VecAzureOpenAIConfig, Text2VecCohereConfig, Text2VecContextionaryConfig, + Text2VecDatabricksConfig, Text2VecGPT4AllConfig, Text2VecHuggingFaceConfig, Text2VecJinaConfig, @@ -140,6 +141,8 @@ export type Text2VecCohereConfigCreate = Text2VecCohereConfig; export type Text2VecContextionaryConfigCreate = Text2VecContextionaryConfig; +export type Text2VecDatabricksConfigCreate = Text2VecDatabricksConfig; + export type Text2VecGPT4AllConfigCreate = Text2VecGPT4AllConfig; export type Text2VecHuggingFaceConfigCreate = Text2VecHuggingFaceConfig; @@ -176,6 +179,8 @@ export type VectorizerConfigCreateType = V extends 'img2vec-neural' ? Text2VecContextionaryConfigCreate | undefined : V extends 'text2vec-cohere' ? Text2VecCohereConfigCreate | undefined + : V extends 'text2vec-databricks' + ? Text2VecDatabricksConfigCreate : V extends 'text2vec-gpt4all' ? Text2VecGPT4AllConfigCreate | undefined : V extends 'text2vec-huggingface' diff --git a/src/collections/configure/unit.test.ts b/src/collections/configure/unit.test.ts index 9bcadd80..fbd28df3 100644 --- a/src/collections/configure/unit.test.ts +++ b/src/collections/configure/unit.test.ts @@ -704,6 +704,50 @@ describe('Unit testing of the vectorizer factory class', () => { }); }); + it('should create the correct Text2VecDatabricksConfig type with required & defaults', () => { + const config = configure.vectorizer.text2VecDatabricks({ + name: 'test', + endpoint: 'endpoint', + }); + expect(config).toEqual>({ + name: 'test', + vectorIndex: { + name: 'hnsw', + config: undefined, + }, + vectorizer: { + name: 'text2vec-databricks', + config: { + endpoint: 'endpoint', + }, + }, + }); + }); + + it('should create the correct Text2VecDatabricksConfig type with all values', () => { + const config = configure.vectorizer.text2VecDatabricks({ + name: 'test', + endpoint: 'endpoint', + instruction: 'instruction', + vectorizeCollectionName: true, + }); + expect(config).toEqual>({ + name: 'test', + vectorIndex: { + name: 'hnsw', + config: undefined, + }, + vectorizer: { + name: 'text2vec-databricks', + config: { + endpoint: 'endpoint', + instruction: 'instruction', + vectorizeCollectionName: true, + }, + }, + }); + }); + it('should create the correct Text2VecGPT4AllConfig type with defaults', () => { const config = configure.vectorizer.text2VecGPT4All(); expect(config).toEqual>({ diff --git a/src/collections/configure/vectorizer.ts b/src/collections/configure/vectorizer.ts index de9bb7b4..61477951 100644 --- a/src/collections/configure/vectorizer.ts +++ b/src/collections/configure/vectorizer.ts @@ -299,6 +299,27 @@ export const vectorizer = { }, }); }, + /** + * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-databricks'`. + * + * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/databricks/embeddings) for detailed usage. + * + * @param {ConfigureTextVectorizerOptions} opts The configuration for the `text2vec-databricks` vectorizer. + * @returns {VectorConfigCreate, N, I, 'text2vec-databricks'>} The configuration object. + */ + text2VecDatabricks: ( + opts: ConfigureTextVectorizerOptions + ): VectorConfigCreate, N, I, 'text2vec-databricks'> => { + const { name, sourceProperties, vectorIndexConfig, ...config } = opts; + return makeVectorizer(name, { + sourceProperties, + vectorIndexConfig, + vectorizerConfig: { + name: 'text2vec-databricks', + config: config, + }, + }); + }, /** * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-gpt4all'`. *