Migrate to connect v2 and generate with buf

docs/how_to_release.md

@@ -3,7 +3,7 @@
 ## Pre-requisites
 
 -   Install Node.js 18 or higher
--   Insall `pnpm` 9. With never versions, it won't work! Download the binary directly from here: https://github.com/pnpm/pnpm/releases/tag/v9.15.9
+-   Install `pnpm` 10
 
 Install dependencies using `pnpm`:
 

package.json

@@ -6,7 +6,7 @@
         "node": ">=18.17.0",
         "pnpm": ">=8"
     },
-    "packageManager": "pnpm@9.15.9",
+    "packageManager": "pnpm@10.20.0",
     "scripts": {
         "prepare": "node scripts/prepare.cjs && pnpm -r build",
         "ci:version": "pnpm changeset version && pnpm install --no-frozen-lockfile && git add .",

packages/js-client-grpc/buf.gen.yaml

@@ -0,0 +1,10 @@
+version: v2
+inputs:
+  - directory: ./proto
+plugins:
+  - local: protoc-gen-es
+    out: src/proto
+    include_imports: true
+    opt:
+      - target=ts
+      - import_extension=js

packages/js-client-grpc/package.json

@@ -48,14 +48,13 @@
         "codegen:grpc-typescript": "./scripts/generate-grpc-sources.sh"
     },
     "dependencies": {
-        "@bufbuild/connect": "^0.13.0",
-        "@bufbuild/connect-node": "^0.13.0",
-        "@bufbuild/protobuf": "^1.2.1"
+        "@bufbuild/protobuf": "^2.2.2",
+        "@connectrpc/connect": "^2.0.0",
+        "@connectrpc/connect-node": "^2.0.0"
     },
     "devDependencies": {
-        "@bufbuild/protoc-gen-connect-es": "^0.13.0",
-        "@bufbuild/protoc-gen-es": "^1.2.1",
-        "@protobuf-ts/protoc": "^2.9.0",
+        "@bufbuild/buf": "1.59.0",
+        "@bufbuild/protoc-gen-es": "^2.10.0",
         "@sevinf/maybe": "^0.5.0",
         "ts-prune": "^0.10.3",
         "vitest": "^0.31.4"

packages/js-client-grpc/proto/collections.proto

@@ -637,6 +637,7 @@ enum ReplicaState {
   Recovery = 6; // Shard is undergoing recovered by an external node; Normally rejects updates, accepts updates if force is true
   Resharding = 7; // Points are being migrated to this shard as part of scale-up resharding
   ReshardingScaleDown = 8; // Points are being migrated to this shard as part of scale-down resharding
+  ActiveRead = 9; // Active for readers, Partial for writers
 }
 
 message ShardKey {
@@ -740,6 +741,7 @@ message CreateShardKey {
   optional uint32 shards_number = 2; // Number of shards to create per shard key
   optional uint32 replication_factor = 3; // Number of replicas of each shard to create
   repeated uint64 placement = 4; // List of peer ids, allowed to create shards. If empty - all peers are allowed
+  optional ReplicaState initial_state = 5; // Initial state of created replicas. Warning: use with care.
 }
 
 message DeleteShardKey {

packages/js-client-grpc/proto/points.proto

@@ -65,9 +65,9 @@ message InferenceObject {
 }
 
 message Vector {
-  repeated float data = 1; // Vector data (flatten for multi vectors), deprecated
-  optional SparseIndices indices = 2; // Sparse indices for sparse vectors, deprecated
-  optional uint32 vectors_count = 3; // Number of vectors per multi vector, deprecated
+  repeated float data = 1 [deprecated=true]; // Vector data (flatten for multi vectors), deprecated
+  optional SparseIndices indices = 2 [deprecated=true]; // Sparse indices for sparse vectors, deprecated
+  optional uint32 vectors_count = 3 [deprecated=true]; // Number of vectors per multi vector, deprecated
   oneof vector {
     DenseVector dense = 101; // Dense vector
     SparseVector sparse = 102; // Sparse vector
@@ -79,9 +79,9 @@ message Vector {
 }
 
 message VectorOutput {
-  repeated float data = 1; // Vector data (flatten for multi vectors), deprecated
-  optional SparseIndices indices = 2; // Sparse indices for sparse vectors, deprecated
-  optional uint32 vectors_count = 3; // Number of vectors per multi vector, deprecated
+  repeated float data = 1 [deprecated=true]; // Vector data (flatten for multi vectors), deprecated
+  optional SparseIndices indices = 2 [deprecated=true]; // Sparse indices for sparse vectors, deprecated
+  optional uint32 vectors_count = 3 [deprecated=true]; // Number of vectors per multi vector, deprecated
   oneof vector {
     DenseVector dense = 101; // Dense vector
     SparseVector sparse = 102; // Sparse vector

packages/js-client-grpc/scripts/generate-grpc-sources.sh

@@ -36,13 +36,5 @@ cat $CLIENT_DIR/qdrant.proto \
   > $CLIENT_DIR/qdrant_tmp.proto
 mv $CLIENT_DIR/qdrant_tmp.proto $CLIENT_DIR/qdrant.proto
 
-# Generate TypeScript files with protoc plugins
-mkdir -p src/proto && pnpm protoc \
-  -I ./proto \
-  --plugin=protoc-gen-es=./node_modules/@bufbuild/protoc-gen-es/bin/protoc-gen-es \
-  --es_out=./src/proto \
-  --es_opt=target=ts \
-  --plugin=protoc-gen-connect-es=./node_modules/@bufbuild/protoc-gen-connect-es/bin/protoc-gen-connect-es \
-  --connect-es_out=./src/proto \
-  --connect-es_opt=target=ts \
-  ./proto/*.proto
+# Generate TypeScript files with buf
+pnpm buf generate

packages/js-client-grpc/src/api-client.ts

@@ -1,17 +1,17 @@
-import {Code, ConnectError, createPromiseClient, Interceptor, PromiseClient, Transport} from '@bufbuild/connect';
-import {compressionGzip, createGrpcTransport} from '@bufbuild/connect-node';
-import {Collections} from './proto/collections_service_connect.js';
-import {Points} from './proto/points_service_connect.js';
-import {Snapshots} from './proto/snapshots_service_connect.js';
-import {Qdrant} from './proto/qdrant_connect.js';
+import {Code, ConnectError, createClient, Interceptor, Client, Transport} from '@connectrpc/connect';
+import {compressionGzip, createGrpcTransport} from '@connectrpc/connect-node';
+import {Collections} from './proto/collections_service_pb.js';
+import {Points} from './proto/points_service_pb.js';
+import {Snapshots} from './proto/snapshots_service_pb.js';
+import {Qdrant} from './proto/qdrant_pb.js';
 import {PACKAGE_VERSION} from './client-version.js';
 import {QdrantClientResourceExhaustedError} from './errors.js';
 
 type Clients = {
-    collections: PromiseClient<typeof Collections>;
-    points: PromiseClient<typeof Points>;
-    snapshots: PromiseClient<typeof Snapshots>;
-    service: PromiseClient<typeof Qdrant>;
+    collections: Client<typeof Collections>;
+    points: Client<typeof Points>;
+    snapshots: Client<typeof Snapshots>;
+    service: Client<typeof Qdrant>;
 };
 
 export type GrpcClients = Readonly<Clients>;
@@ -24,25 +24,25 @@ function createClients(transport: Transport) {
     return {
         get collections() {
             if (!collections) {
-                collections = createPromiseClient(Collections, transport);
+                collections = createClient(Collections, transport);
             }
             return collections;
         },
         get points() {
             if (!points) {
-                points = createPromiseClient(Points, transport);
+                points = createClient(Points, transport);
             }
             return points;
         },
         get snapshots() {
             if (!snapshots) {
-                snapshots = createPromiseClient(Snapshots, transport);
+                snapshots = createClient(Snapshots, transport);
             }
             return snapshots;
         },
         get service() {
             if (!service) {
-                service = createPromiseClient(Qdrant, transport);
+                service = createClient(Qdrant, transport);
             }
             return service;
         },
@@ -87,8 +87,6 @@ export function createApis(baseUrl: string, {timeout, apiKey}: {timeout: number;
 
     const transport = createGrpcTransport({
         baseUrl,
-        httpVersion: '2',
-        keepSessionAlive: true,
         useBinaryFormat: true,
         sendCompression: compressionGzip,
         acceptCompression: [compressionGzip],

packages/js-client-grpc/src/errors.ts

@@ -1,4 +1,4 @@
-import {Code, ConnectError} from '@bufbuild/connect';
+import {Code, ConnectError} from '@connectrpc/connect';
 
 class CustomError extends Error {
     constructor(message: string) {

packages/js-client-grpc/src/index.ts

@@ -1,12 +1,11 @@
-// Apply proto extensions before re-exporting (must be first)
-import './vector-output-extensions.js';
-
 export {QdrantClient, QdrantClientParams} from './qdrant-client.js';
 export * from './errors.js';
-export {ConnectError, Code as ConnectErrorCode} from '@bufbuild/connect';
-export {PlainMessage, PartialMessage, AnyMessage} from '@bufbuild/protobuf';
+export {ConnectError, Code as ConnectErrorCode} from '@connectrpc/connect';
 export * from './proto/collections_pb.js';
 export * from './proto/json_with_int_pb.js';
 export * from './proto/points_pb.js';
 export * from './proto/qdrant_pb.js';
 export * from './proto/snapshots_service_pb.js';
+
+// Helper functions
+export * from './vector-output-helpers.js';