feat: Add support for RQ (#315)

* feat: add support for RQ

* chore: update documentation

Lint & format

* ci: add 1.32 to test matrix and require it in the RQ test

* chore: lint & format

* ci: extend test matrix

The previous commit only added a new env variable, it is not
included in the test matrix automatically.
v1.31 is now tested for node 22.x and v1.32 against the last 3 supported node versions.

* test: update expected default configuration

Add 'muvera' config for Weaviate 1.31+.
Add 'rq' config for Weaviate 1.32+.
Update expected error string

* test: make the test independent of others

This test fast failing with 'source not well formed' error message,
which is Weaviate's speak for 'I dont think this property is a reference'.
Still not sure why, but I figured it may have to do with how the test-suite is structured;
namely, that each test relies on the state produced/left by the previous test.

To test reference deletion, we instead simply create 2 objects,
reference one from the other and then delete that reference.
No external state.

* chore: lint & format

* refactor: re-use defined union type

* chore: lint & format

Author: bevzzz

.github/workflows/main.yaml

@@ -15,6 +15,7 @@ env:
   WEAVIATE_129: 1.29.8
   WEAVIATE_130: 1.30.7
   WEAVIATE_131: 1.31.0
+  WEAVIATE_132: 1.32.0-rc.1
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -48,9 +49,11 @@ jobs:
           { node: "22.x", weaviate: $WEAVIATE_127},
           { node: "22.x", weaviate: $WEAVIATE_128},
           { node: "22.x", weaviate: $WEAVIATE_129},
-          { node: "18.x", weaviate: $WEAVIATE_130},
-          { node: "20.x", weaviate: $WEAVIATE_130},
-          { node: "22.x", weaviate: $WEAVIATE_130}
+          { node: "22.x", weaviate: $WEAVIATE_130},
+          { node: "22.x", weaviate: $WEAVIATE_131},
+          { node: "18.x", weaviate: $WEAVIATE_132},
+          { node: "20.x", weaviate: $WEAVIATE_132},
+          { node: "22.x", weaviate: $WEAVIATE_132}
         ]
     steps:
     - uses: actions/checkout@v3

src/collections/config/index.ts

@@ -19,6 +19,7 @@ import {
   CollectionConfigUpdate,
   PQConfig,
   QuantizerConfig,
+  RQConfig,
   SQConfig,
   VectorIndexConfig,
   VectorIndexConfigDynamic,
@@ -192,6 +193,9 @@ export class Quantizer {
   static isSQ(config?: QuantizerConfig): config is SQConfig {
     return config?.type === 'sq';
   }
+  static isRQ(config?: QuantizerConfig): config is RQConfig {
+    return config?.type === 'rq';
+  }
 }
 
 export const configGuards = {

src/collections/config/integration.test.ts

@@ -12,10 +12,6 @@ import {
   VectorIndexConfigHNSW,
 } from './types/index.js';
 
-const fail = (msg: string) => {
-  throw new Error(msg);
-};
-
 describe('Testing of the collection.config namespace', () => {
   let client: WeaviateClient;
 
@@ -210,6 +206,30 @@ describe('Testing of the collection.config namespace', () => {
     expect(config.vectorizers.default.vectorizer.name).toEqual('none');
   });
 
+  requireAtLeast(1, 32, 0).it('should be able to get the config of a collection with hnsw-rq', async () => {
+    const collectionName = 'TestCollectionConfigGetHNSWPlusRQ';
+    const collection = await client.collections.create({
+      name: collectionName,
+      vectorizers: weaviate.configure.vectorizer.none({
+        vectorIndexConfig: weaviate.configure.vectorIndex.hnsw({
+          quantizer: weaviate.configure.vectorIndex.quantizer.rq(),
+        }),
+      }),
+    });
+    const config = await collection.config.get();
+
+    const vectorIndexConfig = config.vectorizers.default.indexConfig as VectorIndexConfigHNSW;
+    expect(config.name).toEqual(collectionName);
+    expect(config.generative).toBeUndefined();
+    expect(config.reranker).toBeUndefined();
+    expect(vectorIndexConfig).toBeDefined();
+    expect(vectorIndexConfig.quantizer).toBeDefined();
+    expect(vectorIndexConfig.quantizer?.type).toEqual('rq');
+    expect(config.vectorizers.default.indexType).toEqual('hnsw');
+    expect(config.vectorizers.default.properties).toBeUndefined();
+    expect(config.vectorizers.default.vectorizer.name).toEqual('none');
+  });
+
   it('should be able to get the config of a collection with hnsw-bq', async () => {
     const collectionName = 'TestCollectionConfigGetHNSWPlusBQ';
     const query = () =>

src/collections/config/types/vectorIndex.ts

@@ -9,7 +9,7 @@ export type VectorIndexConfigHNSW = {
   filterStrategy: VectorIndexFilterStrategy;
   flatSearchCutoff: number;
   maxConnections: number;
-  quantizer: PQConfig | BQConfig | SQConfig | undefined;
+  quantizer: QuantizerConfig | undefined;
   skip: boolean;
   vectorCacheMaxObjects: number;
   type: 'hnsw';
@@ -61,6 +61,12 @@ export type PQConfig = {
   type: 'pq';
 };
 
+export type RQConfig = {
+  bits?: number;
+  rescoreLimit?: number;
+  type: 'rq';
+};
+
 export type PQEncoderConfig = {
   type: PQEncoderType;
   distribution: PQEncoderDistribution;
@@ -77,4 +83,4 @@ export type VectorIndexFilterStrategy = 'sweeping' | 'acorn';
 
 export type VectorIndexConfig = VectorIndexConfigHNSW | VectorIndexConfigFlat | VectorIndexConfigDynamic;
 
-export type QuantizerConfig = PQConfig | BQConfig | SQConfig;
+export type QuantizerConfig = PQConfig | BQConfig | SQConfig | RQConfig;

src/collections/config/utils.ts

@@ -46,6 +46,8 @@ import {
   PQEncoderType,
   PropertyConfig,
   PropertyVectorizerConfig,
+  QuantizerConfig,
+  RQConfig,
   ReferenceConfig,
   ReplicationConfig,
   Reranker,
@@ -182,6 +184,16 @@ export const parseVectorIndex = (module: ModuleConfig<VectorIndexType, VectorInd
       },
     };
   }
+  if (QuantizerGuards.isRQCreate(quantizer)) {
+    const { type, ...quant } = quantizer;
+    return {
+      ...conf,
+      rq: {
+        ...quant,
+        enabled: true,
+      },
+    };
+  }
 };
 
 export const parseVectorizerConfig = (config?: VectorizerConfig): any => {
@@ -468,11 +480,13 @@ class ConfigMapping {
       throw new WeaviateDeserializationError(
         'Vector index vector cache max objects was not returned by Weaviate'
       );
-    let quantizer: PQConfig | BQConfig | SQConfig | undefined;
+    let quantizer: QuantizerConfig | undefined;
     if (exists<Record<string, any>>(v.pq) && v.pq.enabled === true) {
       quantizer = ConfigMapping.pq(v.pq);
     } else if (exists<Record<string, any>>(v.bq) && v.bq.enabled === true) {
       quantizer = ConfigMapping.bq(v.bq);
+    } else if (exists<Record<string, any>>(v.rq) && v.rq.enabled === true) {
+      quantizer = ConfigMapping.rq(v.rq);
     } else if (exists<Record<string, any>>(v.sq) && v.sq.enabled === true) {
       quantizer = ConfigMapping.sq(v.sq);
     } else {
@@ -508,6 +522,19 @@ class ConfigMapping {
       type: 'bq',
     };
   }
+  static rq(v?: Record<string, unknown>): RQConfig | undefined {
+    if (v === undefined) throw new WeaviateDeserializationError('RQ was not returned by Weaviate');
+    if (!exists<boolean>(v.enabled))
+      throw new WeaviateDeserializationError('RQ enabled was not returned by Weaviate');
+    if (v.enabled === false) return undefined;
+    const bits = v.bits === undefined ? 6 : (v.bits as number);
+    const rescoreLimit = v.rescoreLimit === undefined ? 20 : (v.rescoreLimit as number);
+    return {
+      bits,
+      rescoreLimit,
+      type: 'rq',
+    };
+  }
   static sq(v?: Record<string, unknown>): SQConfig | undefined {
     if (v === undefined) throw new WeaviateDeserializationError('SQ was not returned by Weaviate');
     if (!exists<boolean>(v.enabled))

src/collections/configure/parsing.ts

@@ -3,6 +3,8 @@ import {
   BQConfigUpdate,
   PQConfigCreate,
   PQConfigUpdate,
+  RQConfigCreate,
+  RQConfigUpdate,
   SQConfigCreate,
   SQConfigUpdate,
 } from './types/index.js';
@@ -13,7 +15,9 @@ type QuantizerConfig =
   | BQConfigCreate
   | BQConfigUpdate
   | SQConfigCreate
-  | SQConfigUpdate;
+  | SQConfigUpdate
+  | RQConfigCreate
+  | RQConfigUpdate;
 
 export class QuantizerGuards {
   static isPQCreate(config?: QuantizerConfig): config is PQConfigCreate {
@@ -34,6 +38,12 @@ export class QuantizerGuards {
   static isSQUpdate(config?: QuantizerConfig): config is SQConfigUpdate {
     return (config as SQConfigUpdate)?.type === 'sq';
   }
+  static isRQCreate(config?: QuantizerConfig): config is RQConfigCreate {
+    return (config as RQConfigCreate)?.type === 'rq';
+  }
+  static isRQUpdate(config?: QuantizerConfig): config is RQConfigUpdate {
+    return (config as RQConfigUpdate)?.type === 'rq';
+  }
 }
 
 export function parseWithDefault<D>(value: D | undefined, defaultValue: D): D {

src/collections/configure/types/vectorIndex.ts

@@ -4,6 +4,7 @@ import {
   PQConfig,
   PQEncoderDistribution,
   PQEncoderType,
+  RQConfig,
   SQConfig,
   VectorDistance,
   VectorIndexConfigDynamic,
@@ -17,6 +18,13 @@ export type QuantizerRecursivePartial<T> = {
   [P in keyof T]: P extends 'type' ? T[P] : RecursivePartial<T[P]> | undefined;
 };
 
+export type RQConfigCreate = QuantizerRecursivePartial<RQConfig>;
+
+export type RQConfigUpdate = {
+  rescoreLimit?: number;
+  type: 'rq';
+};
+
 export type PQConfigCreate = QuantizerRecursivePartial<PQConfig>;
 
 export type PQConfigUpdate = {
@@ -59,7 +67,7 @@ export type VectorIndexConfigHNSWUpdate = {
   ef?: number;
   filterStrategy?: VectorIndexFilterStrategy;
   flatSearchCutoff?: number;
-  quantizer?: PQConfigUpdate | BQConfigUpdate | SQConfigUpdate;
+  quantizer?: PQConfigUpdate | BQConfigUpdate | SQConfigUpdate | RQConfigUpdate;
   vectorCacheMaxObjects?: number;
 };
 
@@ -131,7 +139,7 @@ export type VectorIndexConfigHNSWCreateOptions = {
   /** The maximum number of connections. Default is 64. */
   maxConnections?: number;
   /** The quantizer configuration to use. Use `vectorIndex.quantizer.bq` or `vectorIndex.quantizer.pq` to make one. */
-  quantizer?: PQConfigCreate | BQConfigCreate | SQConfigCreate;
+  quantizer?: PQConfigCreate | BQConfigCreate | SQConfigCreate | RQConfigCreate;
   /** Whether to skip the index. Default is false. */
   skip?: boolean;
   /** The maximum number of objects to cache in the vector cache. Default is 1000000000000. */

src/collections/configure/vectorIndex.ts

@@ -9,6 +9,8 @@ import {
   BQConfigUpdate,
   PQConfigCreate,
   PQConfigUpdate,
+  RQConfigCreate,
+  RQConfigUpdate,
   SQConfigCreate,
   SQConfigUpdate,
   VectorIndexConfigDynamicCreate,
@@ -65,7 +67,6 @@ const configure = {
         ? {
             ...rest,
             distance: distanceMetric,
-            quantizer: rest.quantizer,
           }
         : undefined,
     };
@@ -111,12 +112,26 @@ const configure = {
         type: 'bq',
       };
     },
+    /**
+     * Create an object of type `RQConfigCreate` to be used when defining the quantizer configuration of a vector index.
+     *
+     * @param {number} [options.bits] Number of bits to user per vector element.
+     * @param {number} [options.rescoreLimit] The rescore limit. Default is 1000.
+     * @returns {RQConfigCreate} The object of type `RQConfigCreate`.
+     */
+    rq: (options?: { bits?: number; rescoreLimit?: number }): RQConfigCreate => {
+      return {
+        bits: options?.bits,
+        rescoreLimit: options?.rescoreLimit,
+        type: 'rq',
+      };
+    },
     /**
      * Create an object of type `PQConfigCreate` to be used when defining the quantizer configuration of a vector index.
      *
      * @param {boolean} [options.bitCompression] Whether to use bit compression.
-     * @param {number} [options.centroids] The number of centroids[.
-     * @param {PQEncoderDistribution} ]options.encoder.distribution The encoder distribution.
+     * @param {number} [options.centroids] The number of centroids.
+     * @param {PQEncoderDistribution} [options.encoder.distribution] The encoder distribution.
      * @param {PQEncoderType} [options.encoder.type] The encoder type.
      * @param {number} [options.segments] The number of segments.
      * @param {number} [options.trainingLimit] The training limit.
@@ -194,7 +209,7 @@ const reconfigure = {
    * @param {number} [options.ef] The ef parameter. Default is -1.
    * @param {VectorIndexFilterStrategy} [options.filterStrategy] The filter strategy. Default is 'sweeping'.
    * @param {number} [options.flatSearchCutoff] The flat search cutoff. Default is 40000.
-   * @param {PQConfigUpdate | BQConfigUpdate} [options.quantizer] The quantizer configuration to use. Use `vectorIndex.quantizer.bq` or `vectorIndex.quantizer.pq` to make one.
+   * @param {PQConfigUpdate | BQConfigUpdate | SQConfigUpdate | RQConfigUpdate} [options.quantizer] The quantizer configuration to use. Use `vectorIndex.quantizer.bq` or `vectorIndex.quantizer.pq` to make one.
    * @param {number} [options.vectorCacheMaxObjects] The maximum number of objects to cache in the vector cache. Default is 1000000000000.
    * @returns {ModuleConfig<'hnsw', VectorIndexConfigHNSWUpdate>} The configuration object.
    */
@@ -205,7 +220,7 @@ const reconfigure = {
     ef?: number;
     filterStrategy?: VectorIndexFilterStrategy;
     flatSearchCutoff?: number;
-    quantizer?: PQConfigUpdate | BQConfigUpdate | SQConfigUpdate;
+    quantizer?: PQConfigUpdate | BQConfigUpdate | SQConfigUpdate | RQConfigUpdate;
     vectorCacheMaxObjects?: number;
   }): ModuleConfig<'hnsw', VectorIndexConfigHNSWUpdate> => {
     return {
@@ -225,14 +240,29 @@ const reconfigure = {
      *
      * @param {boolean} [options.cache] Whether to cache the quantizer.
      * @param {number} [options.rescoreLimit] The new rescore limit.
-     * @returns {BQConfigCreate} The configuration object.
+     * @returns {BQConfigUpdate} The configuration object.
      */
     bq: (options?: { cache?: boolean; rescoreLimit?: number }): BQConfigUpdate => {
       return {
         ...options,
         type: 'bq',
       };
     },
+    /**
+     * Create an object of type `RQConfigUpdate` to be used when updating the quantizer configuration of a vector index.
+     *
+     * NOTE: If the vector index already has a quantizer configured, you cannot change its quantizer type; only its values.
+     * So if you want to change the quantizer type, you must recreate the collection.
+     *
+     * @param {number} [options.rescoreLimit] The new rescore limit.
+     * @returns {BQConfigUpdate} The configuration object.
+     */
+    rq: (options?: { rescoreLimit?: number }): RQConfigUpdate => {
+      return {
+        ...options,
+        type: 'rq',
+      };
+    },
     /**
      * Create an object of type `PQConfigUpdate` to be used when updating the quantizer configuration of a vector index.
      *