diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 2d34792e..5ebb76b8 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -15,6 +15,7 @@ env: WEAVIATE_129: 1.29.8 WEAVIATE_130: 1.30.7 WEAVIATE_131: 1.31.0 + WEAVIATE_132: 1.32.0-rc.1 concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -48,9 +49,11 @@ jobs: { node: "22.x", weaviate: $WEAVIATE_127}, { node: "22.x", weaviate: $WEAVIATE_128}, { node: "22.x", weaviate: $WEAVIATE_129}, - { node: "18.x", weaviate: $WEAVIATE_130}, - { node: "20.x", weaviate: $WEAVIATE_130}, - { node: "22.x", weaviate: $WEAVIATE_130} + { node: "22.x", weaviate: $WEAVIATE_130}, + { node: "22.x", weaviate: $WEAVIATE_131}, + { node: "18.x", weaviate: $WEAVIATE_132}, + { node: "20.x", weaviate: $WEAVIATE_132}, + { node: "22.x", weaviate: $WEAVIATE_132} ] steps: - uses: actions/checkout@v3 diff --git a/src/collections/config/index.ts b/src/collections/config/index.ts index e03533af..bdc7ed4e 100644 --- a/src/collections/config/index.ts +++ b/src/collections/config/index.ts @@ -19,6 +19,7 @@ import { CollectionConfigUpdate, PQConfig, QuantizerConfig, + RQConfig, SQConfig, VectorIndexConfig, VectorIndexConfigDynamic, @@ -192,6 +193,9 @@ export class Quantizer { static isSQ(config?: QuantizerConfig): config is SQConfig { return config?.type === 'sq'; } + static isRQ(config?: QuantizerConfig): config is RQConfig { + return config?.type === 'rq'; + } } export const configGuards = { diff --git a/src/collections/config/integration.test.ts b/src/collections/config/integration.test.ts index 68163d5e..0daf96f7 100644 --- a/src/collections/config/integration.test.ts +++ b/src/collections/config/integration.test.ts @@ -12,10 +12,6 @@ import { VectorIndexConfigHNSW, } from './types/index.js'; -const fail = (msg: string) => { - throw new Error(msg); -}; - describe('Testing of the collection.config namespace', () => { let client: WeaviateClient; @@ -210,6 +206,30 @@ describe('Testing of the collection.config namespace', () => { expect(config.vectorizers.default.vectorizer.name).toEqual('none'); }); + requireAtLeast(1, 32, 0).it('should be able to get the config of a collection with hnsw-rq', async () => { + const collectionName = 'TestCollectionConfigGetHNSWPlusRQ'; + const collection = await client.collections.create({ + name: collectionName, + vectorizers: weaviate.configure.vectorizer.none({ + vectorIndexConfig: weaviate.configure.vectorIndex.hnsw({ + quantizer: weaviate.configure.vectorIndex.quantizer.rq(), + }), + }), + }); + const config = await collection.config.get(); + + const vectorIndexConfig = config.vectorizers.default.indexConfig as VectorIndexConfigHNSW; + expect(config.name).toEqual(collectionName); + expect(config.generative).toBeUndefined(); + expect(config.reranker).toBeUndefined(); + expect(vectorIndexConfig).toBeDefined(); + expect(vectorIndexConfig.quantizer).toBeDefined(); + expect(vectorIndexConfig.quantizer?.type).toEqual('rq'); + expect(config.vectorizers.default.indexType).toEqual('hnsw'); + expect(config.vectorizers.default.properties).toBeUndefined(); + expect(config.vectorizers.default.vectorizer.name).toEqual('none'); + }); + it('should be able to get the config of a collection with hnsw-bq', async () => { const collectionName = 'TestCollectionConfigGetHNSWPlusBQ'; const query = () => diff --git a/src/collections/config/types/vectorIndex.ts b/src/collections/config/types/vectorIndex.ts index ddd6ea90..5ccdf881 100644 --- a/src/collections/config/types/vectorIndex.ts +++ b/src/collections/config/types/vectorIndex.ts @@ -9,7 +9,7 @@ export type VectorIndexConfigHNSW = { filterStrategy: VectorIndexFilterStrategy; flatSearchCutoff: number; maxConnections: number; - quantizer: PQConfig | BQConfig | SQConfig | undefined; + quantizer: QuantizerConfig | undefined; skip: boolean; vectorCacheMaxObjects: number; type: 'hnsw'; @@ -61,6 +61,12 @@ export type PQConfig = { type: 'pq'; }; +export type RQConfig = { + bits?: number; + rescoreLimit?: number; + type: 'rq'; +}; + export type PQEncoderConfig = { type: PQEncoderType; distribution: PQEncoderDistribution; @@ -77,4 +83,4 @@ export type VectorIndexFilterStrategy = 'sweeping' | 'acorn'; export type VectorIndexConfig = VectorIndexConfigHNSW | VectorIndexConfigFlat | VectorIndexConfigDynamic; -export type QuantizerConfig = PQConfig | BQConfig | SQConfig; +export type QuantizerConfig = PQConfig | BQConfig | SQConfig | RQConfig; diff --git a/src/collections/config/utils.ts b/src/collections/config/utils.ts index 018fbf70..de729ad3 100644 --- a/src/collections/config/utils.ts +++ b/src/collections/config/utils.ts @@ -46,6 +46,8 @@ import { PQEncoderType, PropertyConfig, PropertyVectorizerConfig, + QuantizerConfig, + RQConfig, ReferenceConfig, ReplicationConfig, Reranker, @@ -182,6 +184,16 @@ export const parseVectorIndex = (module: ModuleConfig { @@ -468,11 +480,13 @@ class ConfigMapping { throw new WeaviateDeserializationError( 'Vector index vector cache max objects was not returned by Weaviate' ); - let quantizer: PQConfig | BQConfig | SQConfig | undefined; + let quantizer: QuantizerConfig | undefined; if (exists>(v.pq) && v.pq.enabled === true) { quantizer = ConfigMapping.pq(v.pq); } else if (exists>(v.bq) && v.bq.enabled === true) { quantizer = ConfigMapping.bq(v.bq); + } else if (exists>(v.rq) && v.rq.enabled === true) { + quantizer = ConfigMapping.rq(v.rq); } else if (exists>(v.sq) && v.sq.enabled === true) { quantizer = ConfigMapping.sq(v.sq); } else { @@ -508,6 +522,19 @@ class ConfigMapping { type: 'bq', }; } + static rq(v?: Record): RQConfig | undefined { + if (v === undefined) throw new WeaviateDeserializationError('RQ was not returned by Weaviate'); + if (!exists(v.enabled)) + throw new WeaviateDeserializationError('RQ enabled was not returned by Weaviate'); + if (v.enabled === false) return undefined; + const bits = v.bits === undefined ? 6 : (v.bits as number); + const rescoreLimit = v.rescoreLimit === undefined ? 20 : (v.rescoreLimit as number); + return { + bits, + rescoreLimit, + type: 'rq', + }; + } static sq(v?: Record): SQConfig | undefined { if (v === undefined) throw new WeaviateDeserializationError('SQ was not returned by Weaviate'); if (!exists(v.enabled)) diff --git a/src/collections/configure/parsing.ts b/src/collections/configure/parsing.ts index 09319424..abd0fced 100644 --- a/src/collections/configure/parsing.ts +++ b/src/collections/configure/parsing.ts @@ -3,6 +3,8 @@ import { BQConfigUpdate, PQConfigCreate, PQConfigUpdate, + RQConfigCreate, + RQConfigUpdate, SQConfigCreate, SQConfigUpdate, } from './types/index.js'; @@ -13,7 +15,9 @@ type QuantizerConfig = | BQConfigCreate | BQConfigUpdate | SQConfigCreate - | SQConfigUpdate; + | SQConfigUpdate + | RQConfigCreate + | RQConfigUpdate; export class QuantizerGuards { static isPQCreate(config?: QuantizerConfig): config is PQConfigCreate { @@ -34,6 +38,12 @@ export class QuantizerGuards { static isSQUpdate(config?: QuantizerConfig): config is SQConfigUpdate { return (config as SQConfigUpdate)?.type === 'sq'; } + static isRQCreate(config?: QuantizerConfig): config is RQConfigCreate { + return (config as RQConfigCreate)?.type === 'rq'; + } + static isRQUpdate(config?: QuantizerConfig): config is RQConfigUpdate { + return (config as RQConfigUpdate)?.type === 'rq'; + } } export function parseWithDefault(value: D | undefined, defaultValue: D): D { diff --git a/src/collections/configure/types/vectorIndex.ts b/src/collections/configure/types/vectorIndex.ts index 4f759a7f..22063a73 100644 --- a/src/collections/configure/types/vectorIndex.ts +++ b/src/collections/configure/types/vectorIndex.ts @@ -4,6 +4,7 @@ import { PQConfig, PQEncoderDistribution, PQEncoderType, + RQConfig, SQConfig, VectorDistance, VectorIndexConfigDynamic, @@ -17,6 +18,13 @@ export type QuantizerRecursivePartial = { [P in keyof T]: P extends 'type' ? T[P] : RecursivePartial | undefined; }; +export type RQConfigCreate = QuantizerRecursivePartial; + +export type RQConfigUpdate = { + rescoreLimit?: number; + type: 'rq'; +}; + export type PQConfigCreate = QuantizerRecursivePartial; export type PQConfigUpdate = { @@ -59,7 +67,7 @@ export type VectorIndexConfigHNSWUpdate = { ef?: number; filterStrategy?: VectorIndexFilterStrategy; flatSearchCutoff?: number; - quantizer?: PQConfigUpdate | BQConfigUpdate | SQConfigUpdate; + quantizer?: PQConfigUpdate | BQConfigUpdate | SQConfigUpdate | RQConfigUpdate; vectorCacheMaxObjects?: number; }; @@ -131,7 +139,7 @@ export type VectorIndexConfigHNSWCreateOptions = { /** The maximum number of connections. Default is 64. */ maxConnections?: number; /** The quantizer configuration to use. Use `vectorIndex.quantizer.bq` or `vectorIndex.quantizer.pq` to make one. */ - quantizer?: PQConfigCreate | BQConfigCreate | SQConfigCreate; + quantizer?: PQConfigCreate | BQConfigCreate | SQConfigCreate | RQConfigCreate; /** Whether to skip the index. Default is false. */ skip?: boolean; /** The maximum number of objects to cache in the vector cache. Default is 1000000000000. */ diff --git a/src/collections/configure/vectorIndex.ts b/src/collections/configure/vectorIndex.ts index a9e8790e..73d6fe5a 100644 --- a/src/collections/configure/vectorIndex.ts +++ b/src/collections/configure/vectorIndex.ts @@ -9,6 +9,8 @@ import { BQConfigUpdate, PQConfigCreate, PQConfigUpdate, + RQConfigCreate, + RQConfigUpdate, SQConfigCreate, SQConfigUpdate, VectorIndexConfigDynamicCreate, @@ -65,7 +67,6 @@ const configure = { ? { ...rest, distance: distanceMetric, - quantizer: rest.quantizer, } : undefined, }; @@ -111,12 +112,26 @@ const configure = { type: 'bq', }; }, + /** + * Create an object of type `RQConfigCreate` to be used when defining the quantizer configuration of a vector index. + * + * @param {number} [options.bits] Number of bits to user per vector element. + * @param {number} [options.rescoreLimit] The rescore limit. Default is 1000. + * @returns {RQConfigCreate} The object of type `RQConfigCreate`. + */ + rq: (options?: { bits?: number; rescoreLimit?: number }): RQConfigCreate => { + return { + bits: options?.bits, + rescoreLimit: options?.rescoreLimit, + type: 'rq', + }; + }, /** * Create an object of type `PQConfigCreate` to be used when defining the quantizer configuration of a vector index. * * @param {boolean} [options.bitCompression] Whether to use bit compression. - * @param {number} [options.centroids] The number of centroids[. - * @param {PQEncoderDistribution} ]options.encoder.distribution The encoder distribution. + * @param {number} [options.centroids] The number of centroids. + * @param {PQEncoderDistribution} [options.encoder.distribution] The encoder distribution. * @param {PQEncoderType} [options.encoder.type] The encoder type. * @param {number} [options.segments] The number of segments. * @param {number} [options.trainingLimit] The training limit. @@ -194,7 +209,7 @@ const reconfigure = { * @param {number} [options.ef] The ef parameter. Default is -1. * @param {VectorIndexFilterStrategy} [options.filterStrategy] The filter strategy. Default is 'sweeping'. * @param {number} [options.flatSearchCutoff] The flat search cutoff. Default is 40000. - * @param {PQConfigUpdate | BQConfigUpdate} [options.quantizer] The quantizer configuration to use. Use `vectorIndex.quantizer.bq` or `vectorIndex.quantizer.pq` to make one. + * @param {PQConfigUpdate | BQConfigUpdate | SQConfigUpdate | RQConfigUpdate} [options.quantizer] The quantizer configuration to use. Use `vectorIndex.quantizer.bq` or `vectorIndex.quantizer.pq` to make one. * @param {number} [options.vectorCacheMaxObjects] The maximum number of objects to cache in the vector cache. Default is 1000000000000. * @returns {ModuleConfig<'hnsw', VectorIndexConfigHNSWUpdate>} The configuration object. */ @@ -205,7 +220,7 @@ const reconfigure = { ef?: number; filterStrategy?: VectorIndexFilterStrategy; flatSearchCutoff?: number; - quantizer?: PQConfigUpdate | BQConfigUpdate | SQConfigUpdate; + quantizer?: PQConfigUpdate | BQConfigUpdate | SQConfigUpdate | RQConfigUpdate; vectorCacheMaxObjects?: number; }): ModuleConfig<'hnsw', VectorIndexConfigHNSWUpdate> => { return { @@ -225,7 +240,7 @@ const reconfigure = { * * @param {boolean} [options.cache] Whether to cache the quantizer. * @param {number} [options.rescoreLimit] The new rescore limit. - * @returns {BQConfigCreate} The configuration object. + * @returns {BQConfigUpdate} The configuration object. */ bq: (options?: { cache?: boolean; rescoreLimit?: number }): BQConfigUpdate => { return { @@ -233,6 +248,21 @@ const reconfigure = { type: 'bq', }; }, + /** + * Create an object of type `RQConfigUpdate` to be used when updating the quantizer configuration of a vector index. + * + * NOTE: If the vector index already has a quantizer configured, you cannot change its quantizer type; only its values. + * So if you want to change the quantizer type, you must recreate the collection. + * + * @param {number} [options.rescoreLimit] The new rescore limit. + * @returns {BQConfigUpdate} The configuration object. + */ + rq: (options?: { rescoreLimit?: number }): RQConfigUpdate => { + return { + ...options, + type: 'rq', + }; + }, /** * Create an object of type `PQConfigUpdate` to be used when updating the quantizer configuration of a vector index. * diff --git a/src/collections/data/integration.test.ts b/src/collections/data/integration.test.ts index 681609a2..904abbfa 100644 --- a/src/collections/data/integration.test.ts +++ b/src/collections/data/integration.test.ts @@ -31,15 +31,11 @@ describe('Testing of the collection.data methods with a single target reference' const toBeDeletedID = v4(); const nonExistingID = v4(); - afterAll(() => { - return client.collections.delete(collectionName).catch((err) => { - console.error(err); - throw err; - }); - }); - beforeAll(async () => { client = await weaviate.connectToLocal(); + + await client.collections.delete(collectionName); + collection = client.collections.use(collectionName); await client.collections .create({ @@ -344,32 +340,58 @@ describe('Testing of the collection.data methods with a single target reference' }); it('should be able to delete a reference between two objects', () => { - return Promise.all([ - collection.data.referenceDelete({ - fromProperty: 'ref', - fromUuid: toBeUpdatedID, - to: Reference.to(existingID), - }), - collection.data.referenceDelete({ - fromProperty: 'ref', - fromUuid: toBeUpdatedID, - to: toBeUpdatedID, - }), - collection.data.referenceDelete({ - fromProperty: 'ref', - fromUuid: toBeUpdatedID, - to: [toBeReplacedID], - }), - ]) - .then(() => - collection.query.fetchObjectById(toBeUpdatedID, { - returnReferences: [{ linkOn: 'ref' }], + // Insert 2 objects + return ( + collection.data + .insertMany([{ testProp: 'refLeft' }, { testProp: 'refRight' }]) + // Create a reference between them + .then((inserted) => + collection.data + .referenceAdd({ + fromProperty: 'ref', + fromUuid: inserted.allResponses[0] as string, + to: inserted.allResponses[1] as string, + }) + .then(() => inserted) + ) + + // Assert that the reference exists + .then((inserted) => + collection.query + .fetchObjectById(inserted.allResponses[0] as string, { + returnReferences: [{ linkOn: 'ref' }], + }) + .then((obj) => { + expect(obj).not.toBeNull(); + expect(obj?.references?.ref?.objects).toHaveLength(1); + + // Propagate the list of inserted IDs + return Promise.resolve(inserted); + }) + ) + + // Delete reference between them + .then((inserted) => + collection.data + .referenceDelete({ + fromProperty: 'ref', + fromUuid: inserted.allResponses[0] as string, + to: inserted.allResponses[1] as string, + }) + .then(() => inserted) + ) + + // Assert the reference does not exist + .then((inserted) => + collection.query.fetchObjectById(inserted.allResponses[0] as string, { + returnReferences: [{ linkOn: 'ref' }], + }) + ) + .then((obj) => { + expect(obj).not.toBeNull(); + expect(obj?.references?.ref?.objects).toEqual([]); }) - ) - .then((obj) => { - expect(obj).not.toBeNull(); - expect(obj?.references?.ref?.objects).toEqual([]); - }); + ); }); it('should be able to add many references in batch', () => { diff --git a/src/schema/journey.test.ts b/src/schema/journey.test.ts index eb67dc5f..b1a3d411 100644 --- a/src/schema/journey.test.ts +++ b/src/schema/journey.test.ts @@ -104,8 +104,8 @@ describe('schema', () => { .withProperty(prop) .do() .catch((err: Error) => { - expect(err.message).toEqual( - 'The request to Weaviate failed with status code: 422 and message: {"error":[{"message":"Tokenization is not allowed for data type \'int[]\'"}]}' + expect(err.message.toLowerCase()).toEqual( + 'The request to Weaviate failed with status code: 422 and message: {"error":[{"message":"tokenization is not allowed for data type \'int[]\'"}]}'.toLowerCase() ); }); }); @@ -736,6 +736,16 @@ async function newClassObject(className: string, client: WeaviateClient): Promis ? { aggregation: 'maxSim', enabled: false, + ...((await isVer(client, 31, 0)) + ? { + muvera: { + enabled: false, + dprojections: 16, + ksim: 4, + repetitions: 10, + }, + } + : {}), } : undefined, pq: { @@ -759,6 +769,13 @@ async function newClassObject(className: string, client: WeaviateClient): Promis trainingLimit: 100000, } : undefined, + rq: (await isVer(client, 32, 0)) + ? { + enabled: false, + bits: 8, + rescoreLimit: 20, + } + : undefined, skip: false, efConstruction: 128, vectorCacheMaxObjects: 500000,