Make MongoDB vector field configurable (#160)

* Make MongoDB vector field configurable

* Support multiple vector fields

* Add README

* comment lint

* make it required and the only path

Author: superseb

README.md

@@ -299,11 +299,12 @@ docker run --net=host --rm -it registry.cloud.qdrant.io/library/qdrant-migration
 
 #### MongoDB Options
 
-| Flag                   | Description               |
-| ---------------------- | ------------------------- |
-| `--mongodb.url`        | MongoDB connection string |
-| `--mongodb.database`   | MongoDB database name     |
-| `--mongodb.collection` | MongoDB collection name   |
+| Flag                      | Description                                           |
+| ------------------------- | ----------------------------------------------------- |
+| `--mongodb.url`           | MongoDB connection string                             |
+| `--mongodb.database`      | MongoDB database name                                 |
+| `--mongodb.collection`    | MongoDB collection name                               |
+| `--mongodb.vector-fields` | MongoDB vector fields, comma separated **(required)** |
 
 #### Qdrant Options
 

cmd/migrate_from_mongo.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"slices"
 	"syscall"
 
 	"github.com/pterm/pterm"
@@ -19,10 +20,11 @@ import (
 )
 
 type MigrateFromMongoDBCmd struct {
-	MongoDB   commons.MongoDBConfig   `embed:"" prefix:"mongodb."`
-	Qdrant    commons.QdrantConfig    `embed:"" prefix:"qdrant."`
-	Migration commons.MigrationConfig `embed:"" prefix:"migration."`
-	IdField   string                  `prefix:"qdrant." help:"Field storing MongoDB IDs in Qdrant." default:"__id__"`
+	MongoDB      commons.MongoDBConfig   `embed:"" prefix:"mongodb."`
+	Qdrant       commons.QdrantConfig    `embed:"" prefix:"qdrant."`
+	Migration    commons.MigrationConfig `embed:"" prefix:"migration."`
+	IdField      string                  `prefix:"qdrant." help:"Field storing MongoDB IDs in Qdrant." default:"__id__"`
+	VectorFields []string                `required:"true" prefix:"mongodb." help:"Fields to use as vector."`
 
 	targetHost string
 	targetPort int
@@ -186,9 +188,12 @@ func (r *MigrateFromMongoDBCmd) migrateData(ctx context.Context, sourceClient *m
 				if fieldName == "_id" {
 					continue
 				}
-
-				if vector, ok := extractVector(value); ok {
-					vectors[fieldName] = qdrant.NewVector(vector...)
+				if slices.Contains(r.VectorFields, fieldName) {
+					if vector, ok := extractVector(value); ok {
+						vectors[fieldName] = qdrant.NewVector(vector...)
+					} else {
+						payload[fieldName] = value
+					}
 				} else {
 					payload[fieldName] = value
 				}

integration_tests/migrate_from_mongo_test.go

@@ -13,6 +13,13 @@ import (
 	"github.com/qdrant/go-client/qdrant"
 )
 
+const (
+	// Test multiple vectors.
+	vectorFieldText  = "text_embedding"
+	vectorFieldImage = "image_embedding"
+	nonVectorField   = "non_vector"
+)
+
 func TestMigrateFromMongo(t *testing.T) {
 	ctx := context.Background()
 
@@ -44,22 +51,30 @@ func TestMigrateFromMongo(t *testing.T) {
 	coll := db.Collection("testcoll")
 
 	testIDs := make([]string, totalEntries)
-	testVectors := make([][]float32, totalEntries)
+	testVectorsText := make([][]float32, totalEntries)
+	testVectorsImage := make([][]float32, totalEntries)
 	testDocs := make([]string, totalEntries)
 	testSources := make([]string, totalEntries)
 
+	randomVectorPoints := randFloat32Values(dimension)
+	// Test with additional non vector array
+	nonVectorArray := []float32{1.0, 2.0, 3.0, 4.0, 5.0}
+
 	for i := 0; i < totalEntries; i++ {
 		testIDs[i] = fmt.Sprintf("%d", i+1)
-		testVectors[i] = randFloat32Values(dimension)
+		testVectorsText[i] = randomVectorPoints
+		testVectorsImage[i] = randomVectorPoints
 		testDocs[i] = fmt.Sprintf("test doc %d", i+1)
 		testSources[i] = fmt.Sprintf("source%d", i+1)
 		_, err := coll.InsertOne(ctx, bson.M{
 			// _id is a mandatory field in MongoDB, so we use it to store the ID.
 			// If not specified, MongoDB will generate a random ObjectID.
-			"_id":    testIDs[i],
-			"vector": testVectors[i],
-			"doc":    testDocs[i],
-			"source": testSources[i],
+			"_id":            testIDs[i],
+			vectorFieldText:  testVectorsText[i],
+			vectorFieldImage: testVectorsImage[i],
+			"doc":            testDocs[i],
+			"source":         testSources[i],
+			nonVectorField:   nonVectorArray,
 		})
 		require.NoError(t, err)
 	}
@@ -77,7 +92,11 @@ func TestMigrateFromMongo(t *testing.T) {
 		CollectionName: testCollectionName,
 		VectorsConfig: qdrant.NewVectorsConfigMap(
 			map[string]*qdrant.VectorParams{
-				"vector": {
+				vectorFieldText: {
+					Size:     uint64(dimension),
+					Distance: qdrant.Distance_Dot,
+				},
+				vectorFieldImage: {
 					Size:     uint64(dimension),
 					Distance: qdrant.Distance_Dot,
 				},
@@ -95,6 +114,7 @@ func TestMigrateFromMongo(t *testing.T) {
 		fmt.Sprintf("--qdrant.api-key=%s", qdrantAPIKey),
 		fmt.Sprintf("--qdrant.collection=%s", testCollectionName),
 		fmt.Sprintf("--qdrant.id-field=%s", idField),
+		fmt.Sprintf("--mongodb.vector-fields=%s", fmt.Sprintf("%s,%s", vectorFieldText, vectorFieldImage)),
 	}
 
 	runMigrationBinary(t, args)
@@ -109,19 +129,25 @@ func TestMigrateFromMongo(t *testing.T) {
 	require.Len(t, points, len(testIDs))
 
 	expectedPoints := make(map[string]struct {
-		doc    string
-		source string
-		vector []float32
+		doc              string
+		source           string
+		vector_text      []float32
+		vector_image     []float32
+		non_vector_array []float32
 	})
 	for i, id := range testIDs {
 		expectedPoints[id] = struct {
-			doc    string
-			source string
-			vector []float32
+			doc              string
+			source           string
+			vector_text      []float32
+			vector_image     []float32
+			non_vector_array []float32
 		}{
-			doc:    testDocs[i],
-			source: testSources[i],
-			vector: testVectors[i],
+			doc:              testDocs[i],
+			source:           testSources[i],
+			vector_text:      testVectorsText[i],
+			vector_image:     testVectorsImage[i],
+			non_vector_array: nonVectorArray,
 		}
 	}
 
@@ -131,7 +157,18 @@ func TestMigrateFromMongo(t *testing.T) {
 		require.True(t, exists)
 		require.Equal(t, expected.doc, point.Payload["doc"].GetStringValue())
 		require.Equal(t, expected.source, point.Payload["source"].GetStringValue())
-		vector := point.Vectors.GetVectors().GetVectors()["vector"].GetData()
-		require.Equal(t, expected.vector, vector)
+		vectorText := point.Vectors.GetVectors().GetVectors()[vectorFieldText].GetData()
+		require.Equal(t, expected.vector_text, vectorText)
+		vectorImage := point.Vectors.GetVectors().GetVectors()[vectorFieldImage].GetData()
+		require.Equal(t, expected.vector_image, vectorImage)
+		nonVectorPayload := point.Payload[nonVectorField].GetListValue()
+		nonVectorArray := make([]float32, len(nonVectorPayload.GetValues()))
+		for i, val := range nonVectorPayload.GetValues() {
+			nonVectorArray[i] = float32(val.GetDoubleValue())
+		}
+		require.Equal(t, expected.non_vector_array, nonVectorArray, "The non-vector array payload does not match the inserted data")
+		// The named vector map should NOT contain nonVectorField
+		_, nonVectorExistsAsVector := point.Vectors.GetVectors().GetVectors()[nonVectorField]
+		require.False(t, nonVectorExistsAsVector, "%s should NOT exist as a named vector", nonVectorField)
 	}
 }