From 295035f40e1bac0dd19c922dcced7f1aac2d3be1 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Thu, 28 Aug 2025 14:35:39 +0200 Subject: [PATCH 1/4] chore: add uncompressed vectorizer factory --- src/collections/config/index.ts | 12 ++++-- src/collections/config/types/vectorIndex.ts | 6 ++- src/collections/config/utils.ts | 45 ++++++++++++--------- src/collections/configure/unit.test.ts | 15 +++++++ src/collections/configure/vectorIndex.ts | 36 +++++++++-------- src/collections/integration.test.ts | 18 +++++++++ 6 files changed, 92 insertions(+), 40 deletions(-) diff --git a/src/collections/config/index.ts b/src/collections/config/index.ts index bdc7ed4e..a535f960 100644 --- a/src/collections/config/index.ts +++ b/src/collections/config/index.ts @@ -21,6 +21,7 @@ import { QuantizerConfig, RQConfig, SQConfig, + UncompressedConfig, VectorIndexConfig, VectorIndexConfigDynamic, VectorIndexConfigFlat, @@ -41,7 +42,7 @@ const config = ( .withClassName(name) .withProperty(resolveProperty(property, [])) .do() - .then(() => {}), + .then(() => { }), addReference: ( reference: ReferenceSingleTargetConfigCreate | ReferenceMultiTargetConfigCreate ) => @@ -49,7 +50,7 @@ const config = ( .withClassName(name) .withProperty(resolveReference(reference)) .do() - .then(() => {}), + .then(() => { }), addVector: async (vectors: VectorizersConfigAdd) => { const supportsDynamicVectorIndex = await dbVersionSupport.supportsDynamicVectorIndex(); const { vectorsConfig } = makeVectorsConfig(vectors, supportsDynamicVectorIndex); @@ -73,7 +74,7 @@ const config = ( }) ); }, - updateShards: async function (status: 'READY' | 'READONLY', names?: string | string[]) { + updateShards: async function(status: 'READY' | 'READONLY', names?: string | string[]) { let shardNames: string[]; if (names === undefined) { shardNames = await this.getShards().then((shards) => shards.map((s) => s.name)); @@ -98,7 +99,7 @@ const config = ( ) ) .then((merged) => new ClassUpdater(connection).withClass(merged).do()) - .then(() => {}); + .then(() => { }); }, }; }; @@ -184,6 +185,9 @@ export class VectorIndex { } export class Quantizer { + static isUncompressed(config?: QuantizerConfig): config is UncompressedConfig { + return config?.type === 'none'; + } static isPQ(config?: QuantizerConfig): config is PQConfig { return config?.type === 'pq'; } diff --git a/src/collections/config/types/vectorIndex.ts b/src/collections/config/types/vectorIndex.ts index b9c894cf..9229e6da 100644 --- a/src/collections/config/types/vectorIndex.ts +++ b/src/collections/config/types/vectorIndex.ts @@ -68,6 +68,10 @@ export type RQConfig = { type: 'rq'; }; +export type UncompressedConfig = { + type: 'none'; +} + export type MultiVectorConfig = { aggregation: 'maxSim' | string; encoding?: MultiVectorEncodingConfig; @@ -98,4 +102,4 @@ export type VectorIndexFilterStrategy = 'sweeping' | 'acorn'; export type VectorIndexConfig = VectorIndexConfigHNSW | VectorIndexConfigFlat | VectorIndexConfigDynamic; -export type QuantizerConfig = PQConfig | BQConfig | SQConfig | RQConfig; +export type QuantizerConfig = PQConfig | BQConfig | SQConfig | RQConfig | UncompressedConfig; diff --git a/src/collections/config/utils.ts b/src/collections/config/utils.ts index 76104422..33f197b8 100644 --- a/src/collections/config/utils.ts +++ b/src/collections/config/utils.ts @@ -30,6 +30,7 @@ import { VectorizersConfigAdd, VectorizersConfigCreate, } from '../configure/types/index.js'; +import { Quantizer } from '../index.js'; import { BQConfig, CollectionConfig, @@ -179,6 +180,12 @@ export const parseVectorIndex = (module: ModuleConfig { if (v.vectorIndex.name === 'dynamic' && !supportsDynamicVectorIndex.supports) { throw new WeaviateUnsupportedFeatureError(supportsDynamicVectorIndex.message); @@ -366,18 +373,18 @@ class ConfigMapping { vectorizer: v.vectorizer === 'none' ? { - name: 'none', - config: undefined, - } + name: 'none', + config: undefined, + } : { - name: v.vectorizer, - config: v.moduleConfig - ? ({ - ...(v.moduleConfig[v.vectorizer] as any), - vectorizeCollectionName: (v.moduleConfig[v.vectorizer] as any).vectorizeClassName, - } as VectorizerConfig) - : undefined, - }, + name: v.vectorizer, + config: v.moduleConfig + ? ({ + ...(v.moduleConfig[v.vectorizer] as any), + vectorizeCollectionName: (v.moduleConfig[v.vectorizer] as any).vectorizeClassName, + } as VectorizerConfig) + : undefined, + }, indexConfig: ConfigMapping.vectorIndex(v.vectorIndexConfig, v.vectorIndexType), indexType: ConfigMapping.vectorIndexType(v.vectorIndexType), }, @@ -555,9 +562,9 @@ class ConfigMapping { ) { encoding = v.muvera.enabled ? { - type: 'muvera', - ...v.muvera, - } + type: 'muvera', + ...v.muvera, + } : undefined; } return { diff --git a/src/collections/configure/unit.test.ts b/src/collections/configure/unit.test.ts index e7e61292..82f6c1c2 100644 --- a/src/collections/configure/unit.test.ts +++ b/src/collections/configure/unit.test.ts @@ -255,6 +255,21 @@ describe('Unit testing of the configure & reconfigure factory classes', () => { }); }); + it('should create an hnsw VectorIndexConfig type with "none" quantizer', () => { + const config = configure.vectorIndex.hnsw({ + quantizer: configure.vectorIndex.quantizer.none(), + }); + expect(config).toEqual>({ + name: 'hnsw', + config: { + quantizer: { + type: 'none', + }, + type: 'hnsw', + }, + }); + }); + it('should create an hnsw VectorIndexConfig type with multivector enabled', () => { const config = configure.vectorIndex.hnsw({ multiVector: configure.vectorIndex.multiVector.multiVector({ aggregation: 'maxSim' }), diff --git a/src/collections/configure/vectorIndex.ts b/src/collections/configure/vectorIndex.ts index 83f0eac9..273086f0 100644 --- a/src/collections/configure/vectorIndex.ts +++ b/src/collections/configure/vectorIndex.ts @@ -2,6 +2,7 @@ import { ModuleConfig, PQEncoderDistribution, PQEncoderType, + UncompressedConfig, VectorIndexFilterStrategy, } from '../config/types/index.js'; import { @@ -68,10 +69,10 @@ const configure = { name: 'hnsw', config: rest ? { - ...rest, - distance: distanceMetric, - type: 'hnsw', - } + ...rest, + distance: distanceMetric, + type: 'hnsw', + } : undefined, }; }, @@ -90,12 +91,12 @@ const configure = { name: 'dynamic', config: opts ? { - distance: opts.distanceMetric, - threshold: opts.threshold, - hnsw: isModuleConfig(opts.hnsw) ? opts.hnsw.config : configure.hnsw(opts.hnsw).config, - flat: isModuleConfig(opts.flat) ? opts.flat.config : configure.flat(opts.flat).config, - type: 'dynamic', - } + distance: opts.distanceMetric, + threshold: opts.threshold, + hnsw: isModuleConfig(opts.hnsw) ? opts.hnsw.config : configure.hnsw(opts.hnsw).config, + flat: isModuleConfig(opts.flat) ? opts.flat.config : configure.flat(opts.flat).config, + type: 'dynamic', + } : undefined, }; }, @@ -149,6 +150,9 @@ const configure = { * Define the quantizer configuration to use when creating a vector index. */ quantizer: { + none: (): UncompressedConfig => { + return { type: 'none' }; + }, /** * Create an object of type `BQConfigCreate` to be used when defining the quantizer configuration of a vector index. * @@ -203,9 +207,9 @@ const configure = { centroids: options?.centroids, encoder: options?.encoder ? { - distribution: options.encoder.distribution, - type: options.encoder.type, - } + distribution: options.encoder.distribution, + type: options.encoder.type, + } : undefined, segments: options?.segments, trainingLimit: options?.trainingLimit, @@ -340,9 +344,9 @@ const reconfigure = { encoder: pqEncoderDistribution || pqEncoderType ? { - distribution: pqEncoderDistribution, - type: pqEncoderType, - } + distribution: pqEncoderDistribution, + type: pqEncoderType, + } : undefined, type: 'pq', }; diff --git a/src/collections/integration.test.ts b/src/collections/integration.test.ts index 905f83ce..1b0714ca 100644 --- a/src/collections/integration.test.ts +++ b/src/collections/integration.test.ts @@ -1,4 +1,5 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ +import { requireAtLeast } from '../../test/version'; import weaviate, { WeaviateClient } from '../index'; import { CollectionConfigCreate, @@ -656,6 +657,23 @@ describe('Testing of the collections.create method', () => { ).toEqual(true); }); + requireAtLeast(1, 32, 0).it('should be able to create a collection with uncompressed vectors', async () => { + const collectionName = 'TestCollectionUncompressedVector'; + + await contextionary.collections.create({ + name: collectionName, + vectorizers: weaviate.configure.vectors.text2VecContextionary({ + vectorIndexConfig: weaviate.configure.vectorIndex.hnsw({ + quantizer: weaviate.configure.vectorIndex.quantizer.none(), + }), + }), + }); + + const collection = await contextionary.collections.export(collectionName); + + expect((collection.vectorizers.default.indexConfig as VectorIndexConfigHNSW).quantizer).toBeUndefined(); + }); + it('should be able to create a collection with an openai vectorizer with configure.vectors', async () => { const collectionName = 'TestCollectionOpenAIVectorizerWithConfigureVectorizer'; const response = await openai.collections From c48a27829f5afa444b804d553d65e3baf4f4874d Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Thu, 28 Aug 2025 14:40:00 +0200 Subject: [PATCH 2/4] test: add named vector test case --- src/collections/integration.test.ts | 40 +++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/src/collections/integration.test.ts b/src/collections/integration.test.ts index 1b0714ca..d427811f 100644 --- a/src/collections/integration.test.ts +++ b/src/collections/integration.test.ts @@ -657,21 +657,41 @@ describe('Testing of the collections.create method', () => { ).toEqual(true); }); - requireAtLeast(1, 32, 0).it('should be able to create a collection with uncompressed vectors', async () => { - const collectionName = 'TestCollectionUncompressedVector'; + requireAtLeast(1, 32, 0).describe('uncompressed vectors', () => { + it('should be able to create a collection with uncompressed default vector', async () => { + const collectionName = 'TestCollectionUncompressedVector'; - await contextionary.collections.create({ - name: collectionName, - vectorizers: weaviate.configure.vectors.text2VecContextionary({ - vectorIndexConfig: weaviate.configure.vectorIndex.hnsw({ - quantizer: weaviate.configure.vectorIndex.quantizer.none(), + await contextionary.collections.create({ + name: collectionName, + vectorizers: weaviate.configure.vectors.text2VecContextionary({ + vectorIndexConfig: weaviate.configure.vectorIndex.hnsw({ + quantizer: weaviate.configure.vectorIndex.quantizer.none(), + }), }), - }), + }); + + const collection = await contextionary.collections.export(collectionName); + + expect((collection.vectorizers.default.indexConfig as VectorIndexConfigHNSW).quantizer).toBeUndefined(); }); - const collection = await contextionary.collections.export(collectionName); + it('should be able to create a collection with uncompressed named vector', async () => { + const collectionName = 'TestCollectionUncompressedVectorNamed'; - expect((collection.vectorizers.default.indexConfig as VectorIndexConfigHNSW).quantizer).toBeUndefined(); + await contextionary.collections.create({ + name: collectionName, + vectorizers: weaviate.configure.vectors.text2VecContextionary({ + name: 'custom_vec', + vectorIndexConfig: weaviate.configure.vectorIndex.hnsw({ + quantizer: weaviate.configure.vectorIndex.quantizer.none(), + }), + }), + }); + + const collection = await contextionary.collections.export(collectionName); + + expect((collection.vectorizers['custom_vec'].indexConfig as VectorIndexConfigHNSW).quantizer).toBeUndefined(); + }); }); it('should be able to create a collection with an openai vectorizer with configure.vectors', async () => { From 87678892c986b878e69cf490d8ea82ab171cc6a9 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Thu, 28 Aug 2025 14:41:00 +0200 Subject: [PATCH 3/4] chore: lint&format --- src/collections/config/index.ts | 8 ++--- src/collections/config/types/vectorIndex.ts | 2 +- src/collections/config/utils.ts | 40 ++++++++++----------- src/collections/configure/vectorIndex.ts | 37 ++++++++++--------- src/collections/integration.test.ts | 4 ++- 5 files changed, 49 insertions(+), 42 deletions(-) diff --git a/src/collections/config/index.ts b/src/collections/config/index.ts index a535f960..a8850d17 100644 --- a/src/collections/config/index.ts +++ b/src/collections/config/index.ts @@ -42,7 +42,7 @@ const config = ( .withClassName(name) .withProperty(resolveProperty(property, [])) .do() - .then(() => { }), + .then(() => {}), addReference: ( reference: ReferenceSingleTargetConfigCreate | ReferenceMultiTargetConfigCreate ) => @@ -50,7 +50,7 @@ const config = ( .withClassName(name) .withProperty(resolveReference(reference)) .do() - .then(() => { }), + .then(() => {}), addVector: async (vectors: VectorizersConfigAdd) => { const supportsDynamicVectorIndex = await dbVersionSupport.supportsDynamicVectorIndex(); const { vectorsConfig } = makeVectorsConfig(vectors, supportsDynamicVectorIndex); @@ -74,7 +74,7 @@ const config = ( }) ); }, - updateShards: async function(status: 'READY' | 'READONLY', names?: string | string[]) { + updateShards: async function (status: 'READY' | 'READONLY', names?: string | string[]) { let shardNames: string[]; if (names === undefined) { shardNames = await this.getShards().then((shards) => shards.map((s) => s.name)); @@ -99,7 +99,7 @@ const config = ( ) ) .then((merged) => new ClassUpdater(connection).withClass(merged).do()) - .then(() => { }); + .then(() => {}); }, }; }; diff --git a/src/collections/config/types/vectorIndex.ts b/src/collections/config/types/vectorIndex.ts index 9229e6da..907fdc08 100644 --- a/src/collections/config/types/vectorIndex.ts +++ b/src/collections/config/types/vectorIndex.ts @@ -70,7 +70,7 @@ export type RQConfig = { export type UncompressedConfig = { type: 'none'; -} +}; export type MultiVectorConfig = { aggregation: 'maxSim' | string; diff --git a/src/collections/config/utils.ts b/src/collections/config/utils.ts index 33f197b8..b87d8c16 100644 --- a/src/collections/config/utils.ts +++ b/src/collections/config/utils.ts @@ -184,7 +184,7 @@ export const parseVectorIndex = (module: ModuleConfig { if (v.vectorIndex.name === 'dynamic' && !supportsDynamicVectorIndex.supports) { throw new WeaviateUnsupportedFeatureError(supportsDynamicVectorIndex.message); @@ -373,18 +373,18 @@ class ConfigMapping { vectorizer: v.vectorizer === 'none' ? { - name: 'none', - config: undefined, - } + name: 'none', + config: undefined, + } : { - name: v.vectorizer, - config: v.moduleConfig - ? ({ - ...(v.moduleConfig[v.vectorizer] as any), - vectorizeCollectionName: (v.moduleConfig[v.vectorizer] as any).vectorizeClassName, - } as VectorizerConfig) - : undefined, - }, + name: v.vectorizer, + config: v.moduleConfig + ? ({ + ...(v.moduleConfig[v.vectorizer] as any), + vectorizeCollectionName: (v.moduleConfig[v.vectorizer] as any).vectorizeClassName, + } as VectorizerConfig) + : undefined, + }, indexConfig: ConfigMapping.vectorIndex(v.vectorIndexConfig, v.vectorIndexType), indexType: ConfigMapping.vectorIndexType(v.vectorIndexType), }, @@ -562,9 +562,9 @@ class ConfigMapping { ) { encoding = v.muvera.enabled ? { - type: 'muvera', - ...v.muvera, - } + type: 'muvera', + ...v.muvera, + } : undefined; } return { diff --git a/src/collections/configure/vectorIndex.ts b/src/collections/configure/vectorIndex.ts index 273086f0..39bd8af3 100644 --- a/src/collections/configure/vectorIndex.ts +++ b/src/collections/configure/vectorIndex.ts @@ -69,10 +69,10 @@ const configure = { name: 'hnsw', config: rest ? { - ...rest, - distance: distanceMetric, - type: 'hnsw', - } + ...rest, + distance: distanceMetric, + type: 'hnsw', + } : undefined, }; }, @@ -91,12 +91,12 @@ const configure = { name: 'dynamic', config: opts ? { - distance: opts.distanceMetric, - threshold: opts.threshold, - hnsw: isModuleConfig(opts.hnsw) ? opts.hnsw.config : configure.hnsw(opts.hnsw).config, - flat: isModuleConfig(opts.flat) ? opts.flat.config : configure.flat(opts.flat).config, - type: 'dynamic', - } + distance: opts.distanceMetric, + threshold: opts.threshold, + hnsw: isModuleConfig(opts.hnsw) ? opts.hnsw.config : configure.hnsw(opts.hnsw).config, + flat: isModuleConfig(opts.flat) ? opts.flat.config : configure.flat(opts.flat).config, + type: 'dynamic', + } : undefined, }; }, @@ -150,6 +150,11 @@ const configure = { * Define the quantizer configuration to use when creating a vector index. */ quantizer: { + /** + * Create an object of type `UncomplressedConfig` to skip default vector quantization. + * + * @returns {BQConfigCreate} The object of type `BQConfigCreate`. + */ none: (): UncompressedConfig => { return { type: 'none' }; }, @@ -207,9 +212,9 @@ const configure = { centroids: options?.centroids, encoder: options?.encoder ? { - distribution: options.encoder.distribution, - type: options.encoder.type, - } + distribution: options.encoder.distribution, + type: options.encoder.type, + } : undefined, segments: options?.segments, trainingLimit: options?.trainingLimit, @@ -344,9 +349,9 @@ const reconfigure = { encoder: pqEncoderDistribution || pqEncoderType ? { - distribution: pqEncoderDistribution, - type: pqEncoderType, - } + distribution: pqEncoderDistribution, + type: pqEncoderType, + } : undefined, type: 'pq', }; diff --git a/src/collections/integration.test.ts b/src/collections/integration.test.ts index d427811f..193d8bec 100644 --- a/src/collections/integration.test.ts +++ b/src/collections/integration.test.ts @@ -690,7 +690,9 @@ describe('Testing of the collections.create method', () => { const collection = await contextionary.collections.export(collectionName); - expect((collection.vectorizers['custom_vec'].indexConfig as VectorIndexConfigHNSW).quantizer).toBeUndefined(); + expect( + (collection.vectorizers.custom_vec.indexConfig as VectorIndexConfigHNSW).quantizer + ).toBeUndefined(); }); }); From c14359036ebb146b27bfb398921c9a76e9594dee Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Thu, 28 Aug 2025 14:42:33 +0200 Subject: [PATCH 4/4] ci: target appropriate server version for 1.32 --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 5ebb76b8..5e70eab5 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -15,7 +15,7 @@ env: WEAVIATE_129: 1.29.8 WEAVIATE_130: 1.30.7 WEAVIATE_131: 1.31.0 - WEAVIATE_132: 1.32.0-rc.1 + WEAVIATE_132: 1.32.4-cdf9a3b concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}