From b8ce749f9a9c72158f35b74dc950ccac2fa815d3 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Tue, 26 Aug 2025 11:31:15 +0100 Subject: [PATCH 1/4] Add `bm25Operator` to aggregate hybrid args type --- src/collections/aggregate/index.ts | 3 ++- src/collections/aggregate/integration.test.ts | 13 ++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/collections/aggregate/index.ts b/src/collections/aggregate/index.ts index a4cce65e..9060ad8d 100644 --- a/src/collections/aggregate/index.ts +++ b/src/collections/aggregate/index.ts @@ -9,7 +9,7 @@ import { WeaviateInvalidInputError, WeaviateQueryError } from '../../errors.js'; import { Aggregator } from '../../graphql/index.js'; import { PrimitiveKeys, toBase64FromMedia } from '../../index.js'; import { Deserialize } from '../deserialize/index.js'; -import { Bm25QueryProperty, NearVectorInputType, TargetVector } from '../query/types.js'; +import { Bm25OperatorOptions, Bm25QueryProperty, NearVectorInputType, TargetVector } from '../query/types.js'; import { NearVectorInputGuards } from '../query/utils.js'; import { Serialize } from '../serialize/index.js'; @@ -45,6 +45,7 @@ export type AggregateHybridOptions = AggregateBaseOptions & { queryProperties?: (PrimitiveKeys | Bm25QueryProperty)[]; targetVector?: TargetVector; vector?: number[]; + bm25Operator?: Bm25OperatorOptions; }; export type AggregateGroupByHybridOptions = AggregateHybridOptions & { diff --git a/src/collections/aggregate/integration.test.ts b/src/collections/aggregate/integration.test.ts index cd64d09a..df4cc30e 100644 --- a/src/collections/aggregate/integration.test.ts +++ b/src/collections/aggregate/integration.test.ts @@ -1,7 +1,8 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ /* eslint-disable @typescript-eslint/no-non-null-asserted-optional-chain */ +import { requireAtLeast } from '../../../test/version.js'; import { WeaviateQueryError, WeaviateUnsupportedFeatureError } from '../../errors.js'; -import weaviate, { AggregateText, WeaviateClient } from '../../index.js'; +import weaviate, { AggregateText, Bm25Operator, WeaviateClient } from '../../index.js'; import { Collection } from '../collection/index.js'; import { CrossReference } from '../references/index.js'; import { DataObject } from '../types/index.js'; @@ -448,6 +449,16 @@ describe('Testing of collection.aggregate search methods', () => { expect(result.totalCount).toBeGreaterThan(0); }); + requireAtLeast(1, 31, 0).it('bm25 search operator with hybrid', async () => { + const result = await collection.aggregate.hybrid('test', { + bm25Operator: Bm25Operator.and(), + maxVectorDistance: 1, + queryProperties: ['text'], + returnMetrics: collection.metrics.aggregate('text').text(['count']), + }); + expect(result.totalCount).toBeGreaterThan(0); + }); + it('should return a grouped aggregation on a hybrid search', async () => { if (await client.getWeaviateVersion().then((ver) => ver.isLowerThan(1, 25, 0))) { console.warn('Skipping test as there is a bug with this in 1.24.26 that will not be fixed'); From 2668580f4491b599f54fde65e7436beb3a21b09a Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Thu, 28 Aug 2025 13:16:35 +0100 Subject: [PATCH 2/4] Allow users to specify an uncompressed quantizer to bypass default quantization --- .github/workflows/main.yaml | 10 +++---- src/collections/config/integration.test.ts | 24 +++++++++++++++++ src/collections/config/types/vectorIndex.ts | 4 +++ src/collections/config/utils.ts | 6 +++++ src/collections/configure/parsing.ts | 5 +++- src/collections/configure/types/base.ts | 7 +++-- .../configure/types/vectorIndex.ts | 26 ++++++++++++++++--- src/collections/configure/vectorIndex.ts | 10 +++++++ 8 files changed, 79 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 5ebb76b8..78db4a34 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -7,15 +7,14 @@ on: pull_request: env: - WEAVIATE_124: 1.24.26 WEAVIATE_125: 1.25.34 WEAVIATE_126: 1.26.17 WEAVIATE_127: 1.27.27 WEAVIATE_128: 1.28.16 - WEAVIATE_129: 1.29.8 - WEAVIATE_130: 1.30.7 - WEAVIATE_131: 1.31.0 - WEAVIATE_132: 1.32.0-rc.1 + WEAVIATE_129: 1.29.9 + WEAVIATE_130: 1.30.12 + WEAVIATE_131: 1.31.5 + WEAVIATE_132: 1.32.4-cdf9a3b concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -43,7 +42,6 @@ jobs: fail-fast: false matrix: versions: [ - { node: "22.x", weaviate: $WEAVIATE_124}, { node: "22.x", weaviate: $WEAVIATE_125}, { node: "22.x", weaviate: $WEAVIATE_126}, { node: "22.x", weaviate: $WEAVIATE_127}, diff --git a/src/collections/config/integration.test.ts b/src/collections/config/integration.test.ts index a640dcbb..e5d3af64 100644 --- a/src/collections/config/integration.test.ts +++ b/src/collections/config/integration.test.ts @@ -2,6 +2,7 @@ import { requireAtLeast } from '../../../test/version.js'; import { WeaviateUnsupportedFeatureError } from '../../errors.js'; import weaviate, { WeaviateClient, weaviateV2 } from '../../index.js'; +import { WeaviateClass } from '../../openapi/types.js'; import { GenerativeCohereConfig, ModuleConfig, @@ -833,4 +834,27 @@ describe('Testing of the collection.config namespace', () => { expect(indexConfig.multiVector?.encoding).toBeUndefined(); } ); + + requireAtLeast(1, 32, 4).it.only( + 'should be able to create a collection with an uncompressed quantizer', + async () => { + const collectionName = 'TestCollectionConfigCreateWithUncompressedQuantizer'; + const collection = await client.collections.create({ + name: collectionName, + vectorizers: weaviate.configure.vectors.selfProvided({ + quantizer: weaviate.configure.vectorIndex.quantizer.none(), + }), + }); + await collection.config + .get() + .then((config) => + expect((config.vectorizers.default.indexConfig as VectorIndexConfigHNSW).quantizer).toBeUndefined() + ); + await fetch(`http://localhost:8080/v1/schema/${collectionName}`) + .then((res) => res.json() as WeaviateClass) + .then((schema) => + expect(schema.vectorConfig?.default.vectorIndexConfig?.skipDefaultQuantization).toBe(true) + ); + } + ); }); diff --git a/src/collections/config/types/vectorIndex.ts b/src/collections/config/types/vectorIndex.ts index b9c894cf..7d257048 100644 --- a/src/collections/config/types/vectorIndex.ts +++ b/src/collections/config/types/vectorIndex.ts @@ -68,6 +68,10 @@ export type RQConfig = { type: 'rq'; }; +export type UncompressedConfig = { + type: 'none'; +}; + export type MultiVectorConfig = { aggregation: 'maxSim' | string; encoding?: MultiVectorEncodingConfig; diff --git a/src/collections/config/utils.ts b/src/collections/config/utils.ts index 76104422..17b3dec5 100644 --- a/src/collections/config/utils.ts +++ b/src/collections/config/utils.ts @@ -219,6 +219,12 @@ export const parseVectorIndex = (module: ModuleConfig { diff --git a/src/collections/configure/parsing.ts b/src/collections/configure/parsing.ts index caeaaa3f..30b6b1e6 100644 --- a/src/collections/configure/parsing.ts +++ b/src/collections/configure/parsing.ts @@ -1,4 +1,4 @@ -import { MuveraEncodingConfigCreate } from '../index.js'; +import { MuveraEncodingConfigCreate, UncompressedConfigCreate } from '../index.js'; import { BQConfigCreate, BQConfigUpdate, @@ -49,6 +49,9 @@ export class QuantizerGuards { static isRQUpdate(config?: QuantizerConfig): config is RQConfigUpdate { return (config as RQConfigUpdate)?.type === 'rq'; } + static isUncompressedCreate(config?: QuantizerConfig): config is UncompressedConfigCreate { + return (config as UncompressedConfigCreate)?.type === 'none'; + } } type VectorIndexConfigCreate = diff --git a/src/collections/configure/types/base.ts b/src/collections/configure/types/base.ts index 4754f952..6009bbb6 100644 --- a/src/collections/configure/types/base.ts +++ b/src/collections/configure/types/base.ts @@ -2,15 +2,18 @@ import { WeaviateNestedProperty, WeaviateProperty } from '../../../openapi/types import { InvertedIndexConfig, MultiTenancyConfig, + QuantizerConfig, ReplicationConfig, ReplicationDeletionStrategy, } from '../../config/types/index.js'; -import { DataType } from '../../types/index.js'; +import { DataType, QuantizerRecursivePartial } from '../../types/index.js'; import { NonRefKeys, RefKeys } from '../../types/internal.js'; export type RecursivePartial = T extends object ? { - [P in keyof T]?: RecursivePartial; + [P in keyof T]?: T[P] extends QuantizerConfig + ? QuantizerRecursivePartial + : RecursivePartial; } : T; diff --git a/src/collections/configure/types/vectorIndex.ts b/src/collections/configure/types/vectorIndex.ts index b7c7624a..30423373 100644 --- a/src/collections/configure/types/vectorIndex.ts +++ b/src/collections/configure/types/vectorIndex.ts @@ -8,6 +8,7 @@ import { PQEncoderType, RQConfig, SQConfig, + UncompressedConfig, VectorDistance, VectorIndexConfigDynamic, VectorIndexConfigFlat, @@ -57,11 +58,14 @@ export type SQConfigUpdate = { type: 'sq'; }; +export type UncompressedConfigCreate = QuantizerRecursivePartial; + export type QuantizerConfigCreate = | PQConfigCreate | BQConfigCreate | SQConfigCreate | RQConfigCreate + | UncompressedConfigCreate | Record; export type QuantizerConfigUpdate = @@ -80,11 +84,23 @@ export type MuveraEncodingConfigCreate = RecursivePartial; export type MultiVectorEncodingConfigCreate = MuveraEncodingConfigCreate; -export type VectorIndexConfigHNSWCreate = RecursivePartial; +export type VectorIndexConfigHNSWCreate = RecursivePartial> & { + quantizer?: QuantizerConfigCreate; +}; -export type VectorIndexConfigDynamicCreate = RecursivePartial; +export type VectorIndexConfigDynamicCreate = RecursivePartial< + Omit +> & { + hnsw?: VectorIndexConfigHNSWCreate; + flat?: VectorIndexConfigFlatCreate; +}; -export type VectorIndexConfigDymamicUpdate = RecursivePartial; +export type VectorIndexConfigDymamicUpdate = RecursivePartial< + Omit +> & { + hnsw?: VectorIndexConfigHNSWUpdate; + flat?: VectorIndexConfigFlatUpdate; +}; export type VectorIndexConfigHNSWUpdate = { dynamicEfMin?: number; @@ -107,7 +123,9 @@ export type VectorIndexConfigCreateType = I extends 'hnsw' ? Record : never; -export type VectorIndexConfigFlatCreate = RecursivePartial; +export type VectorIndexConfigFlatCreate = RecursivePartial> & { + quantizer?: QuantizerConfigCreate; +}; export type VectorIndexConfigFlatUpdate = { quantizer?: BQConfigUpdate; diff --git a/src/collections/configure/vectorIndex.ts b/src/collections/configure/vectorIndex.ts index 83f0eac9..85e007d1 100644 --- a/src/collections/configure/vectorIndex.ts +++ b/src/collections/configure/vectorIndex.ts @@ -149,6 +149,16 @@ const configure = { * Define the quantizer configuration to use when creating a vector index. */ quantizer: { + /** + * Create an object of type `NoneConfigCreate` to be used when defining the quantizer configuration of a vector index. + * + * This is useful for disabling the default quantization present in Weaviate>=1.33.0. + */ + none: () => { + return { + type: 'none', + }; + }, /** * Create an object of type `BQConfigCreate` to be used when defining the quantizer configuration of a vector index. * From fe199f2326c61dc9b0ddfe6a3b997fe6e982f37e Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Thu, 28 Aug 2025 13:27:11 +0100 Subject: [PATCH 3/4] Fix tests for previous versions --- src/collections/config/integration.test.ts | 2 +- src/schema/journey.test.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/collections/config/integration.test.ts b/src/collections/config/integration.test.ts index e5d3af64..3a40c6a1 100644 --- a/src/collections/config/integration.test.ts +++ b/src/collections/config/integration.test.ts @@ -835,7 +835,7 @@ describe('Testing of the collection.config namespace', () => { } ); - requireAtLeast(1, 32, 4).it.only( + requireAtLeast(1, 32, 4).it( 'should be able to create a collection with an uncompressed quantizer', async () => { const collectionName = 'TestCollectionConfigCreateWithUncompressedQuantizer'; diff --git a/src/schema/journey.test.ts b/src/schema/journey.test.ts index b1a3d411..1fca4def 100644 --- a/src/schema/journey.test.ts +++ b/src/schema/journey.test.ts @@ -781,6 +781,8 @@ async function newClassObject(className: string, client: WeaviateClient): Promis vectorCacheMaxObjects: 500000, flatSearchCutoff: 40000, filterStrategy: (await isVer(client, 27, 0)) ? 'sweeping' : undefined, + skipDefaultQuantization: (await isVer(client, 32, 4)) ? false : undefined, + trackDefaultQuantization: (await isVer(client, 32, 4)) ? false : undefined, }, invertedIndexConfig: { cleanupIntervalSeconds: 60, From 79affde9319f773847a8ba5316bcecb7ec009e51 Mon Sep 17 00:00:00 2001 From: Tommy Smith Date: Thu, 28 Aug 2025 13:53:33 +0100 Subject: [PATCH 4/4] Cherry pick changes from #340 --- src/collections/config/integration.test.ts | 32 ++++++++++++++++++---- src/collections/configure/vectorIndex.ts | 11 ++++---- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/collections/config/integration.test.ts b/src/collections/config/integration.test.ts index 3a40c6a1..c77d4fd6 100644 --- a/src/collections/config/integration.test.ts +++ b/src/collections/config/integration.test.ts @@ -835,10 +835,9 @@ describe('Testing of the collection.config namespace', () => { } ); - requireAtLeast(1, 32, 4).it( - 'should be able to create a collection with an uncompressed quantizer', - async () => { - const collectionName = 'TestCollectionConfigCreateWithUncompressedQuantizer'; + requireAtLeast(1, 32, 4).describe('uncompressed quantizer', () => { + it('should be able to create a collection with an uncompressed quantizer', async () => { + const collectionName = 'TestCollectionUncompressedVector'; const collection = await client.collections.create({ name: collectionName, vectorizers: weaviate.configure.vectors.selfProvided({ @@ -855,6 +854,27 @@ describe('Testing of the collection.config namespace', () => { .then((schema) => expect(schema.vectorConfig?.default.vectorIndexConfig?.skipDefaultQuantization).toBe(true) ); - } - ); + }); + + it('should be able to create a collection with uncompressed named vector', async () => { + const collectionName = 'TestCollectionUncompressedVectorNamed'; + const collection = await client.collections.create({ + name: collectionName, + vectorizers: weaviate.configure.vectors.selfProvided({ + name: 'custom', + quantizer: weaviate.configure.vectorIndex.quantizer.none(), + }), + }); + await collection.config + .get() + .then((config) => + expect((config.vectorizers.custom.indexConfig as VectorIndexConfigHNSW).quantizer).toBeUndefined() + ); + await fetch(`http://localhost:8080/v1/schema/${collectionName}`) + .then((res) => res.json() as WeaviateClass) + .then((schema) => + expect(schema.vectorConfig?.custom.vectorIndexConfig?.skipDefaultQuantization).toBe(true) + ); + }); + }); }); diff --git a/src/collections/configure/vectorIndex.ts b/src/collections/configure/vectorIndex.ts index 85e007d1..17d9d64e 100644 --- a/src/collections/configure/vectorIndex.ts +++ b/src/collections/configure/vectorIndex.ts @@ -2,6 +2,7 @@ import { ModuleConfig, PQEncoderDistribution, PQEncoderType, + UncompressedConfig, VectorIndexFilterStrategy, } from '../config/types/index.js'; import { @@ -150,14 +151,14 @@ const configure = { */ quantizer: { /** - * Create an object of type `NoneConfigCreate` to be used when defining the quantizer configuration of a vector index. + * Create an object of type `UncompressedConfig` to be used when defining the quantizer configuration of a vector index. * * This is useful for disabling the default quantization present in Weaviate>=1.33.0. + * + * @returns {UncompressedConfig} The object of type `UncompressedConfig`. */ - none: () => { - return { - type: 'none', - }; + none: (): UncompressedConfig => { + return { type: 'none' }; }, /** * Create an object of type `BQConfigCreate` to be used when defining the quantizer configuration of a vector index.