Make improvements to the modules system (#327)

* Make improvements to the modules system:
- deprecate `text2vec-contextionary`
- add `text2vec-model2vec`, which is a replacement for `text2vec-contextionary`
- add `dimensions` parameter to `text2vec-transformers`

* Add `dimensions` to `text2vec-transformers` test

Author: tsmith023

src/collections/config/types/vectorizer.ts

@@ -39,6 +39,7 @@ export type Vectorizer =
   | 'text2vec-jinaai'
   | 'text2vec-nvidia'
   | 'text2vec-mistral'
+  | 'text2vec-model2vec'
   | 'text2vec-ollama'
   | 'text2vec-openai'
   | Text2VecPalmVectorizer
@@ -522,6 +523,8 @@ export type Text2VecGoogleConfig = {
  * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/transformers/embeddings) for detailed usage.
  */
 export type Text2VecTransformersConfig = {
+  /** The number of dimensions for the generated embeddings. */
+  dimensions?: number;
   /** The inference url to use where API requests should go. You can use either this OR (`passage_inference_url` & `query_inference_url`). */
   inferenceUrl?: string;
   /** The inference url to use where passage API requests should go. You can use either (this AND query_inference_url) OR `inference_url`. */
@@ -566,6 +569,16 @@ export type Text2VecWeaviateConfig = {
   vectorizeCollectionName?: boolean;
 };
 
+/**
+ * The configuration for text vectorization using the Model2Vec module.
+ */
+export type Text2VecModel2Vec = {
+  /** The URL to use where API requests should go. */
+  inferenceURL?: string;
+  /** Whether to vectorize the collection name. */
+  vectorizeCollectionName?: boolean;
+};
+
 export type NoVectorizerConfig = {};
 
 export type VectorizerConfig =
@@ -586,6 +599,7 @@ export type VectorizerConfig =
   | Text2VecGoogleConfig
   | Text2VecGPT4AllConfig
   | Text2VecHuggingFaceConfig
+  | Text2VecModel2Vec
   | Text2VecJinaAIConfig
   | Text2VecOpenAIConfig
   | Text2VecPalmConfig
@@ -636,6 +650,8 @@ export type VectorizerConfigType<V> = V extends 'img2vec-neural'
   ? Text2VecNvidiaConfig | undefined
   : V extends 'text2vec-mistral'
   ? Text2VecMistralConfig | undefined
+  : V extends 'text2vec-model2vec'
+  ? Text2VecModel2Vec | undefined
   : V extends 'text2vec-ollama'
   ? Text2VecOllamaConfig | undefined
   : V extends 'text2vec-openai'

src/collections/configure/types/vectorizer.ts

@@ -14,6 +14,7 @@ import {
   Text2VecHuggingFaceConfig,
   Text2VecJinaAIConfig,
   Text2VecMistralConfig,
+  Text2VecModel2Vec,
   Text2VecNvidiaConfig,
   Text2VecOllamaConfig,
   Text2VecOpenAIConfig,
@@ -273,6 +274,8 @@ export type Text2VecNvidiaConfigCreate = Text2VecNvidiaConfig;
 
 export type Text2VecMistralConfigCreate = Text2VecMistralConfig;
 
+export type Text2VecModel2VecConfigCreate = Text2VecModel2Vec;
+
 export type Text2VecOllamaConfigCreate = Text2VecOllamaConfig;
 
 export type Text2VecOpenAIConfigCreate = Text2VecOpenAIConfig;
@@ -330,6 +333,8 @@ export type VectorizerConfigCreateType<V> = V extends 'img2vec-neural'
   ? Text2VecNvidiaConfigCreate | undefined
   : V extends 'text2vec-mistral'
   ? Text2VecMistralConfigCreate | undefined
+  : V extends 'text2vec-model2vec'
+  ? Text2VecModel2VecConfigCreate | undefined
   : V extends 'text2vec-ollama'
   ? Text2VecOllamaConfigCreate | undefined
   : V extends 'text2vec-openai'

src/collections/configure/unit.test.ts

@@ -1463,6 +1463,7 @@ describe('Unit testing of the vectorizer factory class', () => {
     const config = configure.vectors.text2VecTransformers({
       name: 'test',
       poolingStrategy: 'pooling-strategy',
+      dimensions: 512,
     });
     expect(config).toEqual<VectorConfigCreate<never, 'test', 'hnsw', 'text2vec-transformers'>>({
       name: 'test',
@@ -1474,6 +1475,7 @@ describe('Unit testing of the vectorizer factory class', () => {
         name: 'text2vec-transformers',
         config: {
           poolingStrategy: 'pooling-strategy',
+          dimensions: 512,
         },
       },
     });
@@ -1567,12 +1569,49 @@ describe('Unit testing of the vectorizer factory class', () => {
       },
     });
   });
-});
 
-it('should alias "selfProvided" to "none"', () => {
-  expect(configure.vectors.selfProvided()).toEqual<VectorConfigCreate<never, undefined, 'hnsw', 'none'>>(
-    configure.vectors.none()
-  );
+  it('should alias "selfProvided" to "none"', () => {
+    expect(configure.vectors.selfProvided()).toEqual<VectorConfigCreate<never, undefined, 'hnsw', 'none'>>(
+      configure.vectors.none()
+    );
+  });
+
+  it('should create the correct Text2VecModel2VecConfig type with defaults', () => {
+    const config = configure.vectors.text2VecModel2Vec();
+    expect(config).toEqual<VectorConfigCreate<never, undefined, 'hnsw', 'text2vec-model2vec'>>({
+      name: undefined,
+      vectorIndex: {
+        name: 'hnsw',
+        config: undefined,
+      },
+      vectorizer: {
+        name: 'text2vec-model2vec',
+        config: undefined,
+      },
+    });
+  });
+
+  it('should create the correct Text2VecModel2VecConfig type with all values', () => {
+    const config = configure.vectors.text2VecModel2Vec({
+      name: 'test',
+      inferenceURL: 'url',
+      vectorizeCollectionName: true,
+    });
+    expect(config).toEqual<VectorConfigCreate<never, 'test', 'hnsw', 'text2vec-model2vec'>>({
+      name: 'test',
+      vectorIndex: {
+        name: 'hnsw',
+        config: undefined,
+      },
+      vectorizer: {
+        name: 'text2vec-model2vec',
+        config: {
+          inferenceURL: 'url',
+          vectorizeCollectionName: true,
+        },
+      },
+    });
+  });
 });
 
 describe('Unit testing of the multiVectors factory class', () => {

src/collections/configure/vectorizer.ts

@@ -514,6 +514,7 @@ const legacyVectors = {
    *
    * @param {ConfigureTextVectorizerOptions<T, N, I, 'text2vec-contextionary'>} [opts] The configuration for the `text2vec-contextionary` vectorizer.
    * @returns {VectorConfigCreate<PrimitiveKeys<T>, N, I, 'text2vec-contextionary'>} The configuration object.
+   * @deprecated The contextionary model is old and not recommended for use. If you are looking for a local, lightweight model try the new text2vec-model2vec module instead.
    */
   text2VecContextionary: <T, N extends string | undefined = undefined, I extends VectorIndexType = 'hnsw'>(
     opts?: ConfigureTextVectorizerOptions<T, N, I, 'text2vec-contextionary'>
@@ -556,7 +557,7 @@ const legacyVectors = {
    *
    * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/gpt4all/embeddings) for detailed usage.
    *
-   * @param {ConfigureTextVectorizerOptions<T, N, I, 'text2vec-gpt4all'>} [opts] The configuration for the `text2vec-contextionary` vectorizer.
+   * @param {ConfigureTextVectorizerOptions<T, N, I, 'text2vec-gpt4all'>} [opts] The configuration for the `text2vec-gpt4all` vectorizer.
    * @returns {VectorConfigCreate<PrimitiveKeys<T>, N, I, 'text2vec-gpt4all'>} The configuration object.
    */
   text2VecGPT4All: <T, N extends string | undefined = undefined, I extends VectorIndexType = 'hnsw'>(
@@ -578,7 +579,7 @@ const legacyVectors = {
    *
    * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/huggingface/embeddings) for detailed usage.
    *
-   * @param {ConfigureTextVectorizerOptions<T, N, I, 'text2vec-huggingface'>} [opts] The configuration for the `text2vec-contextionary` vectorizer.
+   * @param {ConfigureTextVectorizerOptions<T, N, I, 'text2vec-huggingface'>} [opts] The configuration for the `text2vec-huggingface` vectorizer.
    * @returns {VectorConfigCreate<PrimitiveKeys<T>, N, I, 'text2vec-huggingface'>} The configuration object.
    */
   text2VecHuggingFace: <T, N extends string | undefined = undefined, I extends VectorIndexType = 'hnsw'>(
@@ -810,6 +811,27 @@ const legacyVectors = {
       },
     });
   },
+
+  /**
+   * Create a `VectorConfigCreate` object with the vectorizer set to `'text2vec-model2vec'`.
+   *
+   * @param {ConfigureTextVectorizerOptions<T, N, I, 'text2vec-model2vec'>} [opts] The configuration for the `text2vec-model2vec` vectorizer.
+   * @returns {VectorConfigCreate<PrimitiveKeys<T>, N, I, 'text2vec-model2vec'>} The configuration object.
+   */
+  text2VecModel2Vec: <T, N extends string | undefined = undefined, I extends VectorIndexType = 'hnsw'>(
+    opts?: ConfigureTextVectorizerOptions<T, N, I, 'text2vec-model2vec'>
+  ): VectorConfigCreate<PrimitiveKeys<T>, N, I, 'text2vec-model2vec'> => {
+    const { name, sourceProperties, quantizer, vectorIndexConfig, ...config } = opts || {};
+    return makeVectorizer(name, {
+      sourceProperties,
+      vectorIndexConfig,
+      quantizer,
+      vectorizerConfig: {
+        name: 'text2vec-model2vec',
+        config: Object.keys(config).length === 0 ? undefined : config,
+      },
+    });
+  },
 };
 
 /** __vectors_shaded modifies some parameters in legacy vectorizer configuration.