Introducing VoyageAI's new multimodal embedding model

Author: fzowl

src/collections/config/types/vectorizer.ts

@@ -190,6 +190,17 @@ export type Multi2VecGoogleConfig = {
  * See the [documentation](https://weaviate.io/developers/weaviate/model-providers/transformers/embeddings-multimodal) for detailed usage.
  */
 export type Multi2VecVoyageAIConfig = {
+  /** The image fields used when vectorizing. */
+  imageFields?: string[];
+  /** The text fields used when vectorizing. */
+  textFields?: string[];
+  /** The weights of the fields used for vectorization. */
+  weights?: {
+    /** The weights of the image fields. */
+    imageFields?: number[];
+    /** The weights of the text fields. */
+    textFields?: number[];
+  };
 };
 
 /** The configuration for reference-based vectorization using the centroid method.

src/collections/configure/types/vectorizer.ts

@@ -151,6 +151,10 @@ export type Multi2VecGoogleConfigCreate = {
 };
 
 export type Multi2VecVoyageAIConfigCreate = {
+  /** The image fields to use in vectorization. Can be string of `Multi2VecField` type. If string, weight 0 will be assumed. */
+  imageFields?: string[] | Multi2VecField[];
+  /** The text fields to use in vectorization. Can be string of `Multi2VecField` type. If string, weight 0 will be assumed. */
+  textFields?: string[] | Multi2VecField[];
 };
 
 export type Ref2VecCentroidConfigCreate = Ref2VecCentroidConfig;

src/collections/configure/vectorizer.ts

@@ -3,6 +3,7 @@ import {
   Multi2VecClipConfig,
   Multi2VecField,
   Multi2VecPalmConfig,
+  Multi2VecVoyageAIConfig,
   VectorIndexType,
   Vectorizer,
   VectorizerConfigType,
@@ -275,12 +276,20 @@ export const vectorizer = {
     opts?: ConfigureNonTextVectorizerOptions<N, I, 'multi2vec-voyageai'>
   ): VectorConfigCreate<never, N, I, 'multi2vec-voyageai'> => {
     const { name, vectorIndexConfig, ...config } = opts || {};
+    const imageFields = config.imageFields?.map(mapMulti2VecField);
+    const textFields = config.textFields?.map(mapMulti2VecField);
+    let weights: Multi2VecVoyageAIConfig['weights'] = {};
+    weights = formatMulti2VecFields(weights, 'imageFields', imageFields);
+    weights = formatMulti2VecFields(weights, 'textFields', textFields);
     return makeVectorizer(name, {
       vectorIndexConfig,
       vectorizerConfig: {
         name: 'multi2vec-voyageai',
         config: {
           ...config,
+          imageFields: imageFields?.map((f) => f.name),
+          textFields: textFields?.map((f) => f.name),
+          weights: Object.keys(weights).length === 0 ? undefined : weights,
         },
       },
     });