Feat/add configure uncompressed quantizer (#339)

* Add `bm25Operator` to aggregate hybrid args type

* Allow users to specify an uncompressed quantizer to bypass default quantization

* Fix tests for previous versions

* Cherry pick changes from #340

Author: tsmith023

.github/workflows/main.yaml

@@ -7,15 +7,14 @@ on:
   pull_request:
 
 env:
-  WEAVIATE_124: 1.24.26
   WEAVIATE_125: 1.25.34
   WEAVIATE_126: 1.26.17
   WEAVIATE_127: 1.27.27
   WEAVIATE_128: 1.28.16
-  WEAVIATE_129: 1.29.8
-  WEAVIATE_130: 1.30.7
-  WEAVIATE_131: 1.31.0
-  WEAVIATE_132: 1.32.0-rc.1
+  WEAVIATE_129: 1.29.9
+  WEAVIATE_130: 1.30.12
+  WEAVIATE_131: 1.31.5
+  WEAVIATE_132: 1.32.4-cdf9a3b
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -43,7 +42,6 @@ jobs:
       fail-fast: false
       matrix:
         versions: [
-          { node: "22.x", weaviate: $WEAVIATE_124},
           { node: "22.x", weaviate: $WEAVIATE_125},
           { node: "22.x", weaviate: $WEAVIATE_126},
           { node: "22.x", weaviate: $WEAVIATE_127},

src/collections/config/integration.test.ts

@@ -2,6 +2,7 @@
 import { requireAtLeast } from '../../../test/version.js';
 import { WeaviateUnsupportedFeatureError } from '../../errors.js';
 import weaviate, { WeaviateClient, weaviateV2 } from '../../index.js';
+import { WeaviateClass } from '../../openapi/types.js';
 import {
   GenerativeCohereConfig,
   ModuleConfig,
@@ -833,4 +834,47 @@ describe('Testing of the collection.config namespace', () => {
       expect(indexConfig.multiVector?.encoding).toBeUndefined();
     }
   );
+
+  requireAtLeast(1, 32, 4).describe('uncompressed quantizer', () => {
+    it('should be able to create a collection with an uncompressed quantizer', async () => {
+      const collectionName = 'TestCollectionUncompressedVector';
+      const collection = await client.collections.create({
+        name: collectionName,
+        vectorizers: weaviate.configure.vectors.selfProvided({
+          quantizer: weaviate.configure.vectorIndex.quantizer.none(),
+        }),
+      });
+      await collection.config
+        .get()
+        .then((config) =>
+          expect((config.vectorizers.default.indexConfig as VectorIndexConfigHNSW).quantizer).toBeUndefined()
+        );
+      await fetch(`http://localhost:8080/v1/schema/${collectionName}`)
+        .then((res) => res.json() as WeaviateClass)
+        .then((schema) =>
+          expect(schema.vectorConfig?.default.vectorIndexConfig?.skipDefaultQuantization).toBe(true)
+        );
+    });
+
+    it('should be able to create a collection with uncompressed named vector', async () => {
+      const collectionName = 'TestCollectionUncompressedVectorNamed';
+      const collection = await client.collections.create({
+        name: collectionName,
+        vectorizers: weaviate.configure.vectors.selfProvided({
+          name: 'custom',
+          quantizer: weaviate.configure.vectorIndex.quantizer.none(),
+        }),
+      });
+      await collection.config
+        .get()
+        .then((config) =>
+          expect((config.vectorizers.custom.indexConfig as VectorIndexConfigHNSW).quantizer).toBeUndefined()
+        );
+      await fetch(`http://localhost:8080/v1/schema/${collectionName}`)
+        .then((res) => res.json() as WeaviateClass)
+        .then((schema) =>
+          expect(schema.vectorConfig?.custom.vectorIndexConfig?.skipDefaultQuantization).toBe(true)
+        );
+    });
+  });
 });

src/collections/config/types/vectorIndex.ts

@@ -68,6 +68,10 @@ export type RQConfig = {
   type: 'rq';
 };
 
+export type UncompressedConfig = {
+  type: 'none';
+};
+
 export type MultiVectorConfig = {
   aggregation: 'maxSim' | string;
   encoding?: MultiVectorEncodingConfig;

src/collections/config/utils.ts

@@ -219,6 +219,12 @@ export const parseVectorIndex = (module: ModuleConfig<VectorIndexType, VectorInd
       },
     };
   }
+  if (QuantizerGuards.isUncompressedCreate(quantizer)) {
+    return {
+      ...conf,
+      skipDefaultQuantization: true,
+    };
+  }
 };
 
 export const parseVectorizerConfig = (config?: VectorizerConfig): any => {

src/collections/configure/parsing.ts

@@ -1,4 +1,4 @@
-import { MuveraEncodingConfigCreate } from '../index.js';
+import { MuveraEncodingConfigCreate, UncompressedConfigCreate } from '../index.js';
 import {
   BQConfigCreate,
   BQConfigUpdate,
@@ -49,6 +49,9 @@ export class QuantizerGuards {
   static isRQUpdate(config?: QuantizerConfig): config is RQConfigUpdate {
     return (config as RQConfigUpdate)?.type === 'rq';
   }
+  static isUncompressedCreate(config?: QuantizerConfig): config is UncompressedConfigCreate {
+    return (config as UncompressedConfigCreate)?.type === 'none';
+  }
 }
 
 type VectorIndexConfigCreate =

src/collections/configure/types/base.ts

@@ -2,15 +2,18 @@ import { WeaviateNestedProperty, WeaviateProperty } from '../../../openapi/types
 import {
   InvertedIndexConfig,
   MultiTenancyConfig,
+  QuantizerConfig,
   ReplicationConfig,
   ReplicationDeletionStrategy,
 } from '../../config/types/index.js';
-import { DataType } from '../../types/index.js';
+import { DataType, QuantizerRecursivePartial } from '../../types/index.js';
 import { NonRefKeys, RefKeys } from '../../types/internal.js';
 
 export type RecursivePartial<T> = T extends object
   ? {
-      [P in keyof T]?: RecursivePartial<T[P]>;
+      [P in keyof T]?: T[P] extends QuantizerConfig
+        ? QuantizerRecursivePartial<T[P]>
+        : RecursivePartial<T[P]>;
     }
   : T;
 

src/collections/configure/types/vectorIndex.ts

@@ -8,6 +8,7 @@ import {
   PQEncoderType,
   RQConfig,
   SQConfig,
+  UncompressedConfig,
   VectorDistance,
   VectorIndexConfigDynamic,
   VectorIndexConfigFlat,
@@ -57,11 +58,14 @@ export type SQConfigUpdate = {
   type: 'sq';
 };
 
+export type UncompressedConfigCreate = QuantizerRecursivePartial<UncompressedConfig>;
+
 export type QuantizerConfigCreate =
   | PQConfigCreate
   | BQConfigCreate
   | SQConfigCreate
   | RQConfigCreate
+  | UncompressedConfigCreate
   | Record<string, any>;
 
 export type QuantizerConfigUpdate =
@@ -80,11 +84,23 @@ export type MuveraEncodingConfigCreate = RecursivePartial<MuveraEncodingConfig>;
 
 export type MultiVectorEncodingConfigCreate = MuveraEncodingConfigCreate;
 
-export type VectorIndexConfigHNSWCreate = RecursivePartial<VectorIndexConfigHNSW>;
+export type VectorIndexConfigHNSWCreate = RecursivePartial<Omit<VectorIndexConfigHNSW, 'quantizer'>> & {
+  quantizer?: QuantizerConfigCreate;
+};
 
-export type VectorIndexConfigDynamicCreate = RecursivePartial<VectorIndexConfigDynamic>;
+export type VectorIndexConfigDynamicCreate = RecursivePartial<
+  Omit<VectorIndexConfigDynamic, 'hnsw' | 'flat'>
+> & {
+  hnsw?: VectorIndexConfigHNSWCreate;
+  flat?: VectorIndexConfigFlatCreate;
+};
 
-export type VectorIndexConfigDymamicUpdate = RecursivePartial<VectorIndexConfigDynamic>;
+export type VectorIndexConfigDymamicUpdate = RecursivePartial<
+  Omit<VectorIndexConfigDynamic, 'hnsw' | 'flat'>
+> & {
+  hnsw?: VectorIndexConfigHNSWUpdate;
+  flat?: VectorIndexConfigFlatUpdate;
+};
 
 export type VectorIndexConfigHNSWUpdate = {
   dynamicEfMin?: number;
@@ -107,7 +123,9 @@ export type VectorIndexConfigCreateType<I> = I extends 'hnsw'
   ? Record<string, any>
   : never;
 
-export type VectorIndexConfigFlatCreate = RecursivePartial<VectorIndexConfigFlat>;
+export type VectorIndexConfigFlatCreate = RecursivePartial<Omit<VectorIndexConfigFlat, 'quantizer'>> & {
+  quantizer?: QuantizerConfigCreate;
+};
 
 export type VectorIndexConfigFlatUpdate = {
   quantizer?: BQConfigUpdate;

src/collections/configure/vectorIndex.ts

@@ -2,6 +2,7 @@ import {
   ModuleConfig,
   PQEncoderDistribution,
   PQEncoderType,
+  UncompressedConfig,
   VectorIndexFilterStrategy,
 } from '../config/types/index.js';
 import {
@@ -149,6 +150,16 @@ const configure = {
    * Define the quantizer configuration to use when creating a vector index.
    */
   quantizer: {
+    /**
+     * Create an object of type `UncompressedConfig` to be used when defining the quantizer configuration of a vector index.
+     *
+     * This is useful for disabling the default quantization present in Weaviate>=1.33.0.
+     *
+     * @returns {UncompressedConfig} The object of type `UncompressedConfig`.
+     */
+    none: (): UncompressedConfig => {
+      return { type: 'none' };
+    },
     /**
      * Create an object of type `BQConfigCreate` to be used when defining the quantizer configuration of a vector index.
      *

src/schema/journey.test.ts

@@ -781,6 +781,8 @@ async function newClassObject(className: string, client: WeaviateClient): Promis
       vectorCacheMaxObjects: 500000,
       flatSearchCutoff: 40000,
       filterStrategy: (await isVer(client, 27, 0)) ? 'sweeping' : undefined,
+      skipDefaultQuantization: (await isVer(client, 32, 4)) ? false : undefined,
+      trackDefaultQuantization: (await isVer(client, 32, 4)) ? false : undefined,
     },
     invertedIndexConfig: {
       cleanupIntervalSeconds: 60,