Update protocol definitions for Qdrant 1.13 (#208)

* Update gRPC protocol buffer definitions

* Generate gRPC types, update builders and conversions

* Add max optimization threads builder

* Add bool index params builder

* Add has vector condition builder

* Add dense, sparse and multi dense vector builders and conversions

* Remove has vector condition builder, follow pattern of other conditions

Author: timvisee

proto/collections.proto

@@ -145,6 +145,17 @@ enum CompressionRatio {
   x64 = 4;
 }
 
+message MaxOptimizationThreads {
+    enum Setting {
+        Auto = 0;
+    }
+
+    oneof variant {
+        uint64 value = 1;
+        Setting setting = 2;
+    }
+}
+
 message OptimizerStatus {
   bool ok = 1;
   string error = 2;
@@ -237,7 +248,7 @@ message OptimizersConfigDiff {
   optional uint64 max_segment_size = 4;
   /*
   Maximum size (in kilobytes) of vectors to store in-memory per segment.
-  Segments larger than this threshold will be stored as read-only memmaped file.
+  Segments larger than this threshold will be stored as read-only memmapped file.
 
   Memmap storage is disabled by default, to enable it, set this threshold to a reasonable value.
 
@@ -260,13 +271,17 @@ message OptimizersConfigDiff {
   Interval between forced flushes.
   */
   optional uint64 flush_interval_sec = 7;
+
+  // Deprecated in favor of `max_optimization_threads`
+  optional uint64 deprecated_max_optimization_threads = 8;
+
   /*
   Max number of threads (jobs) for running optimizations per shard.
   Note: each optimization job will also use `max_indexing_threads` threads by itself for index building.
-  If null - have no limit and choose dynamically to saturate CPU.
+  If "auto" - have no limit and choose dynamically to saturate CPU.
   If 0 - no optimization threads, optimizations will be disabled.
   */
-  optional uint64 max_optimization_threads = 8;
+  optional MaxOptimizationThreads max_optimization_threads = 9;
 }
 
 message ScalarQuantization {
@@ -320,6 +335,13 @@ message StrictModeConfig {
   optional uint32 search_max_hnsw_ef = 6;
   optional bool search_allow_exact  = 7;
   optional float search_max_oversampling  = 8;
+  optional uint64 upsert_max_batchsize  = 9;
+  optional uint64 max_collection_vector_size_bytes  = 10;
+  optional uint32 read_rate_limit = 11; // Max number of read operations per minute per replica
+  optional uint32 write_rate_limit = 12; // Max number of write operations per minute per replica
+  optional uint64 max_collection_payload_size_bytes = 13;
+  optional uint64 filter_max_conditions = 14;
+  optional uint64 condition_max_size = 15;
 }
 
 message CreateCollection {
@@ -351,6 +373,7 @@ message UpdateCollection {
   optional VectorsConfigDiff vectors_config = 6; // New vector parameters
   optional QuantizationConfigDiff quantization_config = 7; // Quantization configuration of vector
   optional SparseVectorConfig sparse_vectors_config = 8; // New sparse vector parameters
+  optional StrictModeConfig strict_mode_config = 9; // New strict mode configuration
 }
 
 message DeleteCollection {
@@ -430,6 +453,7 @@ message TextIndexParams {
 }
 
 message BoolIndexParams {
+    optional bool on_disk = 1; // If true - store index on disk.
 }
 
 message DatetimeIndexParams {
@@ -530,7 +554,8 @@ enum ReplicaState {
   Listener = 4; // A shard which receives data, but is not used for search; Useful for backup shards
   PartialSnapshot = 5; // Deprecated: snapshot shard transfer is in progress; Updates should not be sent to (and are ignored by) the shard
   Recovery = 6; // Shard is undergoing recovered by an external node; Normally rejects updates, accepts updates if force is true
-  Resharding = 7; // Points are being migrated to this shard as part of resharding
+  Resharding = 7; // Points are being migrated to this shard as part of scale-up resharding
+  ReshardingScaleDown = 8; // Points are being migrated to this shard as part of scale-down resharding
 }
 
 message ShardKey {
@@ -566,6 +591,15 @@ message ReshardingInfo {
   uint32 shard_id = 1;
   uint64 peer_id = 2;
   optional ShardKey shard_key = 3;
+  ReshardingDirection direction = 4;
+}
+
+/*
+  Resharding direction, scale up or down in number of shards
+*/
+enum ReshardingDirection {
+  Up = 0; // Scale up, add a new shard
+  Down = 1; // Scale down, remove a shard
 }
 
 message CollectionClusterInfoResponse {
@@ -574,8 +608,7 @@ message CollectionClusterInfoResponse {
   repeated LocalShardInfo local_shards = 3; // Local shards
   repeated RemoteShardInfo remote_shards = 4; // Remote shards
   repeated ShardTransferInfo shard_transfers = 5; // Shard transfers
-  // TODO(resharding): enable on release:
-  // repeated ReshardingInfo resharding_operations = 6; // Resharding operations
+  repeated ReshardingInfo resharding_operations = 6; // Resharding operations
 }
 
 message MoveShard {

proto/points.proto

@@ -46,11 +46,48 @@ message SparseIndices {
   repeated uint32 data = 1;
 }
 
+message Document {
+  string text = 1; // Text of the document
+  string model = 3; // Model name
+  map<string, Value> options = 4; // Model options
+}
+
+message Image {
+  Value image = 1; // Image data, either base64 encoded or URL
+  string model = 2; // Model name
+  map<string, Value> options = 3; // Model options
+}
+
+message InferenceObject {
+  Value object = 1; // Object to infer
+  string model = 2; // Model name
+  map<string, Value> options = 3; // Model options
+}
+
 // Legacy vector format, which determines the vector type by the configuration of its fields.
 message Vector {
-  repeated float data = 1; // Vector data (flatten for multi vectors)
-  optional SparseIndices indices = 2; // Sparse indices for sparse vectors
-  optional uint32 vectors_count = 3; // Number of vectors per multi vector
+  repeated float data = 1; // Vector data (flatten for multi vectors), deprecated
+  optional SparseIndices indices = 2; // Sparse indices for sparse vectors, deprecated
+  optional uint32 vectors_count = 3; // Number of vectors per multi vector, deprecated
+  oneof vector {
+    DenseVector dense = 101; // Dense vector
+    SparseVector sparse = 102; // Sparse vector
+    MultiDenseVector multi_dense = 103; // Multi dense vector
+    Document document = 104;
+    Image image = 105;
+    InferenceObject object = 106;
+  }
+}
+
+message VectorOutput {
+  repeated float data = 1; // Vector data (flatten for multi vectors), deprecated
+  optional SparseIndices indices = 2; // Sparse indices for sparse vectors, deprecated
+  optional uint32 vectors_count = 3; // Number of vectors per multi vector, deprecated
+  oneof vector {
+    DenseVector dense = 101; // Dense vector
+    SparseVector sparse = 102; // Sparse vector
+    MultiDenseVector multi_dense = 103; // Multi dense vector
+  }
 }
 
 message DenseVector {
@@ -73,6 +110,9 @@ message VectorInput {
     DenseVector dense = 2;
     SparseVector sparse = 3;
     MultiDenseVector multi_dense = 4;
+    Document document = 5;
+    Image image = 6;
+    InferenceObject object = 7;
   }
 }
 
@@ -214,13 +254,24 @@ message NamedVectors {
   map<string, Vector> vectors = 1;
 }
 
+message NamedVectorsOutput {
+  map<string, VectorOutput> vectors = 1;
+}
+
 message Vectors {
   oneof vectors_options {
     Vector vector = 1;
     NamedVectors vectors = 2;
   }
 }
 
+message VectorsOutput {
+  oneof vectors_options {
+    VectorOutput vector = 1;
+    NamedVectorsOutput vectors = 2;
+  }
+}
+
 message VectorsSelector {
   repeated string names = 1; // List of vectors to include into result
 }
@@ -737,7 +788,7 @@ message ScoredPoint {
   float score = 3; // Similarity score
   reserved 4; // deprecated "vector" field
   uint64 version = 5; // Last update operation applied to this point
-  optional Vectors vectors = 6; // Vectors to search
+  optional VectorsOutput vectors = 6; // Vectors to search
   optional ShardKey shard_key = 7; // Shard key
   optional OrderValue order_value = 8; // Order by value
 }
@@ -766,21 +817,25 @@ message GroupsResult {
 message SearchResponse {
   repeated ScoredPoint result = 1;
   double time = 2; // Time spent to process
+  optional HardwareUsage usage = 3;
 }
 
 message QueryResponse {
   repeated ScoredPoint result = 1;
   double time = 2; // Time spent to process
+  optional HardwareUsage usage = 3;
 }
 
 message QueryBatchResponse {
   repeated BatchResult result = 1;
   double time = 2; // Time spent to process
+  optional HardwareUsage usage = 3;
 }
 
 message QueryGroupsResponse {
   GroupsResult result = 1;
   double time = 2; // Time spent to process
+  optional HardwareUsage usage = 3;
 }
 
 message BatchResult {
@@ -790,16 +845,19 @@ message BatchResult {
 message SearchBatchResponse {
   repeated BatchResult result = 1;
   double time = 2; // Time spent to process
+  optional HardwareUsage usage = 3;
 }
 
 message SearchGroupsResponse {
   GroupsResult result = 1;
   double time = 2; // Time spent to process
+  optional HardwareUsage usage = 3;
 }
 
 message CountResponse {
   CountResult result = 1;
   double time = 2; // Time spent to process
+  optional HardwareUsage usage = 3;
 }
 
 message ScrollResponse {
@@ -816,7 +874,7 @@ message RetrievedPoint {
   PointId id = 1;
   map<string, Value> payload = 2;
   reserved 3; // deprecated "vector" field
-  optional Vectors vectors = 4;
+  optional VectorsOutput vectors = 4;
   optional ShardKey shard_key = 5; // Shard key
   optional OrderValue order_value = 6; // Order-by value
 }
@@ -829,26 +887,31 @@ message GetResponse {
 message RecommendResponse {
   repeated ScoredPoint result = 1;
   double time = 2; // Time spent to process
+  optional HardwareUsage usage = 3;
 }
 
 message RecommendBatchResponse {
   repeated BatchResult result = 1;
   double time = 2; // Time spent to process
+  optional HardwareUsage usage = 3;
 }
 
 message DiscoverResponse {
   repeated ScoredPoint result = 1;
   double time = 2; // Time spent to process
+  optional HardwareUsage usage = 3;
 }
 
 message DiscoverBatchResponse {
   repeated BatchResult result = 1;
   double time = 2; // Time spent to process
+  optional HardwareUsage usage = 3;
 }
 
 message RecommendGroupsResponse {
   GroupsResult result = 1;
   double time = 2; // Time spent to process
+  optional HardwareUsage usage = 3;
 }
 
 message UpdateBatchResponse {
@@ -864,11 +927,13 @@ message FacetResponse {
 message SearchMatrixPairsResponse {
   SearchMatrixPairs result = 1;
   double time = 2; // Time spent to process
+  optional HardwareUsage usage = 3;
 }
 
 message SearchMatrixOffsetsResponse {
   SearchMatrixOffsets result = 1;
   double time = 2; // Time spent to process
+  optional HardwareUsage usage = 3;
 }
 
 // ---------------------------------------------
@@ -895,6 +960,7 @@ message Condition {
     Filter filter = 4;
     IsNullCondition is_null = 5;
     NestedCondition nested = 6;
+    HasVectorCondition has_vector = 7;
   }
 }
 
@@ -910,6 +976,10 @@ message HasIdCondition {
   repeated PointId has_id = 1;
 }
 
+message HasVectorCondition {
+  string has_vector = 1;
+}
+
 message NestedCondition {
   string key = 1; // Path to nested object
   Filter filter = 2; // Filter condition
@@ -1021,3 +1091,13 @@ message GeoPoint {
   double lon = 1;
   double lat = 2;
 }
+
+// ---------------------------------------------
+// ------------ Hardware measurements ----------
+// ---------------------------------------------
+
+message HardwareUsage {
+  uint64 cpu = 1;
+  uint64 io_read = 2;
+  uint64 io_write = 3;
+}