Skip to content

Support nullable vectors in Java SDK #1730

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
marcelo-cjl wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Support nullable vectors in Java SDK #1730

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
299 changes: 299 additions & 0 deletions examples/src/main/java/io/milvus/v2/NullableVectorExample.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,299 @@
package io.milvus.v2;

import com.google.gson.JsonNull;
import com.google.gson.JsonObject;
import io.milvus.common.utils.JsonUtils;
import io.milvus.v2.client.ConnectConfig;
import io.milvus.v2.client.MilvusClientV2;
import io.milvus.v2.common.DataType;
import io.milvus.v2.common.IndexParam;
import io.milvus.v2.service.collection.request.AddCollectionFieldReq;
import io.milvus.v2.service.collection.request.AddFieldReq;
import io.milvus.v2.service.collection.request.CreateCollectionReq;
import io.milvus.v2.service.collection.request.DropCollectionReq;
import io.milvus.v2.service.collection.request.LoadCollectionReq;
import io.milvus.v2.service.collection.request.ReleaseCollectionReq;
import io.milvus.v2.service.index.request.CreateIndexReq;
import io.milvus.v2.service.vector.request.InsertReq;
import io.milvus.v2.service.vector.request.QueryReq;
import io.milvus.v2.service.vector.request.SearchReq;
import io.milvus.v2.service.vector.request.data.FloatVec;
import io.milvus.v2.service.vector.response.InsertResp;
import io.milvus.v2.service.vector.response.QueryResp;
import io.milvus.v2.service.vector.response.SearchResp;

import java.util.*;

public class NullableVectorExample {
private static final int DIMENSION = 8;
private static final Random RANDOM = new Random();

private static List<Float> generateFloatVector() {
List<Float> vector = new ArrayList<>();
for (int i = 0; i < DIMENSION; i++) {
vector.add(RANDOM.nextFloat());
}
return vector;
}

public static void main(String[] args) throws InterruptedException {
ConnectConfig config = ConnectConfig.builder()
.uri("http://localhost:19530")
.build();
MilvusClientV2 client = new MilvusClientV2(config);
System.out.println("Connected to Milvus\n");

insertNullVectors(client);
addNullableVectorField(client);

client.close(5L);
System.out.println("Done!");
}

private static void insertNullVectors(MilvusClientV2 client) throws InterruptedException {
String collectionName = "java_sdk_example_insert_null_vectors";
System.out.println("=== Demo 1: Insert null vectors ===");

// Drop collection if exists
client.dropCollection(DropCollectionReq.builder()
.collectionName(collectionName)
.build());

// Create collection with nullable vector field
CreateCollectionReq.CollectionSchema schema = CreateCollectionReq.CollectionSchema.builder()
.build();
schema.addField(AddFieldReq.builder()
.fieldName("id")
.dataType(DataType.Int64)
.isPrimaryKey(true)
.autoID(false)
.build());
schema.addField(AddFieldReq.builder()
.fieldName("name")
.dataType(DataType.VarChar)
.maxLength(100)
.build());
schema.addField(AddFieldReq.builder()
.fieldName("embedding")
.dataType(DataType.FloatVector)
.dimension(DIMENSION)
.isNullable(true) // Enable nullable for vector field
.build());

client.createCollection(CreateCollectionReq.builder()
.collectionName(collectionName)
.collectionSchema(schema)
.build());
System.out.println("Created collection with nullable vector field");

// Create index
IndexParam indexParam = IndexParam.builder()
.fieldName("embedding")
.metricType(IndexParam.MetricType.L2)
.indexType(IndexParam.IndexType.FLAT)
.build();
client.createIndex(CreateIndexReq.builder()
.collectionName(collectionName)
.indexParams(Collections.singletonList(indexParam))
.build());

// Load collection
client.loadCollection(LoadCollectionReq.builder()
.collectionName(collectionName)
.build());

// Prepare test data: 100 rows, ~50% null vectors
int totalRows = 100;
int nullPercent = 50;
List<JsonObject> data = new ArrayList<>();
int nullCount = 0;
int validCount = 0;

for (int i = 1; i <= totalRows; i++) {
JsonObject row = new JsonObject();
row.addProperty("id", (long) i);
row.addProperty("name", "item_" + i);

boolean isNull = RANDOM.nextInt(100) < nullPercent;
if (isNull) {
row.add("embedding", JsonNull.INSTANCE);
nullCount++;
} else {
row.add("embedding", JsonUtils.toJsonTree(generateFloatVector()));
validCount++;
}
data.add(row);
}

// Insert data
InsertResp insertResp = client.insert(InsertReq.builder()
.collectionName(collectionName)
.data(data)
.build());
System.out.println("Inserted " + insertResp.getInsertCnt() + " rows: " + validCount + " valid, " + nullCount + " null");

Thread.sleep(1000);

// Query all data
QueryResp queryResp = client.query(QueryReq.builder()
.collectionName(collectionName)
.filter("id >= 0")
.outputFields(Arrays.asList("id", "embedding"))
.limit(totalRows + 10)
.build());

int queryNullCount = 0;
int queryValidCount = 0;
for (QueryResp.QueryResult result : queryResp.getQueryResults()) {
Object embedding = result.getEntity().get("embedding");
if (embedding == null) {
queryNullCount++;
} else {
queryValidCount++;
}
}
System.out.println("Query result: " + queryValidCount + " valid, " + queryNullCount + " null");

// Search - only returns non-null vectors
SearchResp searchResp = client.search(SearchReq.builder()
.collectionName(collectionName)
.data(Collections.singletonList(new FloatVec(generateFloatVector())))
.annsField("embedding")
.topK(10)
.outputFields(Arrays.asList("id", "embedding"))
.build());

List<List<SearchResp.SearchResult>> searchResults = searchResp.getSearchResults();
if (!searchResults.isEmpty()) {
System.out.println("Search returned " + searchResults.get(0).size() + " hits (only non-null vectors)");
}

// Cleanup
client.dropCollection(DropCollectionReq.builder()
.collectionName(collectionName)
.build());
System.out.println("Dropped collection\n");
}

private static void addNullableVectorField(MilvusClientV2 client) throws InterruptedException {
String collectionName = "java_sdk_example_add_vector_field";
System.out.println("=== Demo 2: Add nullable vector field to existing collection ===");

// Drop collection if exists
client.dropCollection(DropCollectionReq.builder()
.collectionName(collectionName)
.build());

// Create collection with one vector field (Milvus requires at least one)
CreateCollectionReq.CollectionSchema schema = CreateCollectionReq.CollectionSchema.builder()
.build();
schema.addField(AddFieldReq.builder()
.fieldName("id")
.dataType(DataType.Int64)
.isPrimaryKey(true)
.autoID(false)
.build());
schema.addField(AddFieldReq.builder()
.fieldName("name")
.dataType(DataType.VarChar)
.maxLength(100)
.build());
schema.addField(AddFieldReq.builder()
.fieldName("embedding_v1")
.dataType(DataType.FloatVector)
.dimension(DIMENSION)
.build());

client.createCollection(CreateCollectionReq.builder()
.collectionName(collectionName)
.collectionSchema(schema)
.build());
System.out.println("Created collection with one vector field");

// Create index and load
IndexParam indexParam = IndexParam.builder()
.fieldName("embedding_v1")
.metricType(IndexParam.MetricType.L2)
.indexType(IndexParam.IndexType.FLAT)
.build();
client.createIndex(CreateIndexReq.builder()
.collectionName(collectionName)
.indexParams(Collections.singletonList(indexParam))
.build());
client.loadCollection(LoadCollectionReq.builder()
.collectionName(collectionName)
.build());

// Insert some data first
List<JsonObject> data = new ArrayList<>();
for (int i = 1; i <= 10; i++) {
JsonObject row = new JsonObject();
row.addProperty("id", (long) i);
row.addProperty("name", "item_" + i);
row.add("embedding_v1", JsonUtils.toJsonTree(generateFloatVector()));
data.add(row);
}
client.insert(InsertReq.builder()
.collectionName(collectionName)
.data(data)
.build());
System.out.println("Inserted 10 rows");

// Release before adding field
client.releaseCollection(ReleaseCollectionReq.builder()
.collectionName(collectionName)
.build());

// Add a second nullable vector field to existing collection
client.addCollectionField(AddCollectionFieldReq.builder()
.collectionName(collectionName)
.fieldName("embedding_v2")
.dataType(DataType.FloatVector)
.dimension(DIMENSION)
.isNullable(true) // Must be nullable when adding to existing collection
.build());
System.out.println("Added nullable vector field 'embedding_v2'");

// Create index for the new field
IndexParam newIndexParam = IndexParam.builder()
.fieldName("embedding_v2")
.metricType(IndexParam.MetricType.L2)
.indexType(IndexParam.IndexType.FLAT)
.build();
client.createIndex(CreateIndexReq.builder()
.collectionName(collectionName)
.indexParams(Collections.singletonList(newIndexParam))
.build());

// Load collection
client.loadCollection(LoadCollectionReq.builder()
.collectionName(collectionName)
.build());

Thread.sleep(1000);

// Query to verify old rows have null for the new field
QueryResp queryResp = client.query(QueryReq.builder()
.collectionName(collectionName)
.filter("id >= 0")
.outputFields(Arrays.asList("id", "embedding_v1", "embedding_v2"))
.limit(10)
.build());

System.out.println("Query result (old rows have null for new field):");
for (QueryResp.QueryResult result : queryResp.getQueryResults()) {
Map<String, Object> entity = result.getEntity();
long id = (Long) entity.get("id");
Object v1 = entity.get("embedding_v1");
Object v2 = entity.get("embedding_v2");
System.out.println(" id=" + id + ", embedding_v1=" + (v1 == null ? "null" : "has value")
+ ", embedding_v2=" + (v2 == null ? "null" : "has value"));
}

// Cleanup
client.dropCollection(DropCollectionReq.builder()
.collectionName(collectionName)
.build());
System.out.println("Dropped collection\n");
}
}
26 changes: 26 additions & 0 deletions sdk-core/src/main/java/io/milvus/param/ParamUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1204,6 +1204,16 @@ public static FieldData genFieldData(String fieldName, DataType dataType, DataTy

FieldData.Builder builder = FieldData.newBuilder();
if (isVectorDataType(dataType)) {
if (isNullable) {
List<Object> tempObjects = new ArrayList<>();
for (Object obj : objects) {
builder.addValidData(obj != null);
if (obj != null) {
tempObjects.add(obj);
}
}
objects = tempObjects;
}
VectorField vectorField = genVectorField(dataType, objects);
return builder.setFieldName(fieldName).setType(dataType).setVectors(vectorField).build();
} else {
Expand All @@ -1228,6 +1238,22 @@ public static FieldData genFieldData(String fieldName, DataType dataType, DataTy

@SuppressWarnings("unchecked")
public static VectorField genVectorField(DataType dataType, List<?> objects) {
if (objects.isEmpty()) {
if (dataType == DataType.FloatVector) {
return VectorField.newBuilder().setDim(0).setFloatVector(FloatArray.newBuilder().build()).build();
} else if (dataType == DataType.BinaryVector) {
return VectorField.newBuilder().setDim(0).setBinaryVector(ByteString.EMPTY).build();
} else if (dataType == DataType.Float16Vector) {
return VectorField.newBuilder().setDim(0).setFloat16Vector(ByteString.EMPTY).build();
} else if (dataType == DataType.BFloat16Vector) {
return VectorField.newBuilder().setDim(0).setBfloat16Vector(ByteString.EMPTY).build();
} else if (dataType == DataType.Int8Vector) {
return VectorField.newBuilder().setDim(0).setInt8Vector(ByteString.EMPTY).build();
} else if (dataType == DataType.SparseFloatVector) {
return VectorField.newBuilder().setDim(0).setSparseFloatVector(SparseFloatArray.newBuilder().build()).build();
}
}

if (dataType == DataType.FloatVector) {
List<Float> floats = new ArrayList<>();
// each object is List<Float>
Expand Down
Loading
Loading