diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 69c5bdd5..335f89dd 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -14,6 +14,7 @@ env: WEAVIATE_128: 1.28.11 WEAVIATE_129: 1.29.1 WEAVIATE_130: 1.30.1 + WEAVIATE_131: 1.31.0 concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} diff --git a/src/collections/config/index.ts b/src/collections/config/index.ts index 320545c7..ee982d3f 100644 --- a/src/collections/config/index.ts +++ b/src/collections/config/index.ts @@ -4,11 +4,13 @@ import { WeaviateShardStatus } from '../../openapi/types.js'; import ClassUpdater from '../../schema/classUpdater.js'; import { ClassGetter, PropertyCreator, ShardUpdater } from '../../schema/index.js'; import ShardsGetter from '../../schema/shardsGetter.js'; +import VectorAdder from '../../schema/vectorAdder.js'; import { DbVersionSupport } from '../../utils/dbVersion.js'; import { PropertyConfigCreate, ReferenceMultiTargetConfigCreate, ReferenceSingleTargetConfigCreate, + VectorizersConfigAdd, } from '../configure/types/index.js'; import { MergeWithExisting } from './classes.js'; import { @@ -23,7 +25,7 @@ import { VectorIndexConfigFlat, VectorIndexConfigHNSW, } from './types/index.js'; -import { classToCollection, resolveProperty, resolveReference } from './utils.js'; +import { classToCollection, makeVectorsConfig, resolveProperty, resolveReference } from './utils.js'; const config = ( connection: Connection, @@ -47,6 +49,11 @@ const config = ( .withProperty(resolveReference(reference)) .do() .then(() => {}), + addVector: async (vectors: VectorizersConfigAdd) => { + const supportsDynamicVectorIndex = await dbVersionSupport.supportsDynamicVectorIndex(); + const { vectorsConfig } = makeVectorsConfig(vectors, supportsDynamicVectorIndex); + return new VectorAdder(connection).withClassName(name).withVectors(vectorsConfig).do(); + }, get: () => getRaw().then(classToCollection), getShards: () => { let builder = new ShardsGetter(connection).withClassName(name); @@ -114,6 +121,17 @@ export interface Config { addReference: ( reference: ReferenceSingleTargetConfigCreate | ReferenceMultiTargetConfigCreate ) => Promise; + /** + * Add one or more named vectors to the collection in Weaviate. + * Named vectors can be added to collections with existing named vectors only. + * + * Existing named vectors are immutable in Weaviate. The client will not include + * any of those in the request. + * + * @param {VectorizersConfigAdd} vectors Vector configurations. + * @returns {Promise} A promise that resolves when the named vector has been created. + */ + addVector: (vectors: VectorizersConfigAdd) => Promise; /** * Get the configuration for this collection from Weaviate. * diff --git a/src/collections/config/integration.test.ts b/src/collections/config/integration.test.ts index be729ea0..9b363d06 100644 --- a/src/collections/config/integration.test.ts +++ b/src/collections/config/integration.test.ts @@ -1,4 +1,5 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ +import { requireAtLeast } from '../../../test/version.js'; import { WeaviateUnsupportedFeatureError } from '../../errors.js'; import weaviate, { WeaviateClient, weaviateV2 } from '../../index.js'; import { @@ -386,6 +387,43 @@ describe('Testing of the collection.config namespace', () => { ]); }); + requireAtLeast( + 1, + 31, + 0 + )('Mutable named vectors', () => { + it('should be able to add named vectors to a collection', async () => { + const collectionName = 'TestCollectionConfigAddVector' as const; + const collection = await client.collections.create({ + name: collectionName, + vectorizers: weaviate.configure.vectorizer.none(), + }); + // Add a single named vector + await collection.config.addVector(weaviate.configure.vectorizer.none({ name: 'vector-a' })); + + // Add several named vectors + await collection.config.addVector([ + weaviate.configure.vectorizer.none({ name: 'vector-b' }), + weaviate.configure.vectorizer.none({ name: 'vector-c' }), + ]); + + // Trying to update 'default' vector -- should be omitted from request. + await collection.config.addVector( + weaviate.configure.vectorizer.none({ + name: 'default', + vectorIndexConfig: weaviate.configure.vectorIndex.flat(), + }) + ); + + const config = await collection.config.get(); + expect(config.vectorizers).toHaveProperty('vector-a'); + expect(config.vectorizers).toHaveProperty('vector-b'); + expect(config.vectorizers).toHaveProperty('vector-c'); + + expect(config.vectorizers.default).toHaveProperty('indexType', 'hnsw'); + }); + }); + it('should get the shards of a sharded collection', async () => { const shards = await client.collections .create({ diff --git a/src/collections/config/utils.ts b/src/collections/config/utils.ts index da6ab9d6..a66a1313 100644 --- a/src/collections/config/utils.ts +++ b/src/collections/config/utils.ts @@ -1,5 +1,10 @@ -import { WeaviateDeserializationError } from '../../errors.js'; import { + WeaviateDeserializationError, + WeaviateInvalidInputError, + WeaviateUnsupportedFeatureError, +} from '../../errors.js'; +import { + Properties, WeaviateBM25Config, WeaviateClass, WeaviateInvertedIndexConfig, @@ -13,11 +18,19 @@ import { WeaviateVectorIndexConfig, WeaviateVectorsConfig, } from '../../openapi/types.js'; +import { DbVersionSupport } from '../../utils/dbVersion.js'; +import { QuantizerGuards } from '../configure/parsing.js'; import { PropertyConfigCreate, ReferenceConfigCreate, ReferenceMultiTargetConfigCreate, ReferenceSingleTargetConfigCreate, + VectorIndexConfigCreate, + VectorIndexConfigDynamicCreate, + VectorIndexConfigFlatCreate, + VectorIndexConfigHNSWCreate, + VectorizersConfigAdd, + VectorizersConfigCreate, } from '../configure/types/index.js'; import { BQConfig, @@ -46,6 +59,7 @@ import { VectorIndexConfigHNSW, VectorIndexConfigType, VectorIndexFilterStrategy, + VectorIndexType, VectorizerConfig, } from './types/index.js'; @@ -123,6 +137,91 @@ export const classToCollection = (cls: WeaviateClass): CollectionConfig => { }; }; +export const parseVectorIndex = (module: ModuleConfig): any => { + if (module.config === undefined) return undefined; + if (module.name === 'dynamic') { + const { hnsw, flat, ...conf } = module.config as VectorIndexConfigDynamicCreate; + return { + ...conf, + hnsw: parseVectorIndex({ name: 'hnsw', config: hnsw }), + flat: parseVectorIndex({ name: 'flat', config: flat }), + }; + } + const { quantizer, ...conf } = module.config as + | VectorIndexConfigFlatCreate + | VectorIndexConfigHNSWCreate + | Record; + if (quantizer === undefined) return conf; + if (QuantizerGuards.isBQCreate(quantizer)) { + const { type, ...quant } = quantizer; + return { + ...conf, + bq: { + ...quant, + enabled: true, + }, + }; + } + if (QuantizerGuards.isPQCreate(quantizer)) { + const { type, ...quant } = quantizer; + return { + ...conf, + pq: { + ...quant, + enabled: true, + }, + }; + } +}; + +export const parseVectorizerConfig = (config?: VectorizerConfig): any => { + if (config === undefined) return {}; + const { vectorizeCollectionName, ...rest } = config as any; + return { + ...rest, + vectorizeClassName: vectorizeCollectionName, + }; +}; + +export const makeVectorsConfig = ( + configVectorizers: VectorizersConfigCreate | VectorizersConfigAdd, + supportsDynamicVectorIndex: Awaited> +) => { + let vectorizers: string[] = []; + const vectorsConfig: Record = {}; + const vectorizersConfig = Array.isArray(configVectorizers) + ? configVectorizers + : [ + { + ...configVectorizers, + name: configVectorizers.name || 'default', + }, + ]; + vectorizersConfig.forEach((v) => { + if (v.vectorIndex.name === 'dynamic' && !supportsDynamicVectorIndex.supports) { + throw new WeaviateUnsupportedFeatureError(supportsDynamicVectorIndex.message); + } + const vectorConfig: any = { + vectorIndexConfig: parseVectorIndex(v.vectorIndex), + vectorIndexType: v.vectorIndex.name, + vectorizer: {}, + }; + const vectorizer = v.vectorizer.name === 'text2vec-azure-openai' ? 'text2vec-openai' : v.vectorizer.name; + vectorizers = [...vectorizers, vectorizer]; + vectorConfig.vectorizer[vectorizer] = { + properties: v.properties, + ...parseVectorizerConfig(v.vectorizer.config), + }; + if (v.name === undefined) { + throw new WeaviateInvalidInputError( + 'vectorName is required for each vectorizer when specifying more than one vectorizer' + ); + } + vectorsConfig[v.name] = vectorConfig; + }); + return { vectorsConfig, vectorizers }; +}; + function populated(v: T | null | undefined): v is T { return v !== undefined && v !== null; } diff --git a/src/collections/configure/types/vectorizer.ts b/src/collections/configure/types/vectorizer.ts index 5391c356..14f32da7 100644 --- a/src/collections/configure/types/vectorizer.ts +++ b/src/collections/configure/types/vectorizer.ts @@ -55,7 +55,11 @@ export type VectorConfigUpdate = - | VectorConfigCreate, undefined, VectorIndexType, Vectorizer> + | VectorConfigCreate, string | undefined, VectorIndexType, Vectorizer> + | VectorConfigCreate, string, VectorIndexType, Vectorizer>[]; + +export type VectorizersConfigAdd = + | VectorConfigCreate, string, VectorIndexType, Vectorizer> | VectorConfigCreate, string, VectorIndexType, Vectorizer>[]; export type ConfigureNonTextVectorizerOptions< diff --git a/src/collections/index.ts b/src/collections/index.ts index b5c2edb5..cd5707a7 100644 --- a/src/collections/index.ts +++ b/src/collections/index.ts @@ -1,12 +1,18 @@ import Connection from '../connection/grpc.js'; -import { WeaviateInvalidInputError, WeaviateUnsupportedFeatureError } from '../errors.js'; +import { WeaviateUnsupportedFeatureError } from '../errors.js'; import { WeaviateClass } from '../openapi/types.js'; import ClassExists from '../schema/classExists.js'; import { ClassCreator, ClassDeleter, ClassGetter, SchemaGetter } from '../schema/index.js'; import { DbVersionSupport } from '../utils/dbVersion.js'; import collection, { Collection } from './collection/index.js'; -import { classToCollection, resolveProperty, resolveReference } from './config/utils.js'; -import { QuantizerGuards } from './configure/parsing.js'; +import { + classToCollection, + makeVectorsConfig, + parseVectorIndex, + parseVectorizerConfig, + resolveProperty, + resolveReference, +} from './config/utils.js'; import { configGuards } from './index.js'; import { CollectionConfig, @@ -23,13 +29,7 @@ import { RerankerConfig, ShardingConfigCreate, VectorConfigCreate, - VectorIndexConfigCreate, - VectorIndexConfigDynamicCreate, - VectorIndexConfigFlatCreate, - VectorIndexConfigHNSWCreate, - VectorIndexType, Vectorizer, - VectorizerConfig, VectorizersConfigCreate, } from './types/index.js'; import { PrimitiveKeys } from './types/internal.js'; @@ -65,52 +65,6 @@ export type CollectionConfigCreate = { vectorizers?: VectorizersConfigCreate; }; -const parseVectorIndex = (module: ModuleConfig): any => { - if (module.config === undefined) return undefined; - if (module.name === 'dynamic') { - const { hnsw, flat, ...conf } = module.config as VectorIndexConfigDynamicCreate; - return { - ...conf, - hnsw: parseVectorIndex({ name: 'hnsw', config: hnsw }), - flat: parseVectorIndex({ name: 'flat', config: flat }), - }; - } - const { quantizer, ...conf } = module.config as - | VectorIndexConfigFlatCreate - | VectorIndexConfigHNSWCreate - | Record; - if (quantizer === undefined) return conf; - if (QuantizerGuards.isBQCreate(quantizer)) { - const { type, ...quant } = quantizer; - return { - ...conf, - bq: { - ...quant, - enabled: true, - }, - }; - } - if (QuantizerGuards.isPQCreate(quantizer)) { - const { type, ...quant } = quantizer; - return { - ...conf, - pq: { - ...quant, - enabled: true, - }, - }; - } -}; - -const parseVectorizerConfig = (config?: VectorizerConfig): any => { - if (config === undefined) return {}; - const { vectorizeCollectionName, ...rest } = config as any; - return { - ...rest, - vectorizeClassName: vectorizeCollectionName, - }; -}; - const collections = (connection: Connection, dbVersionSupport: DbVersionSupport) => { const listAll = () => new SchemaGetter(connection) @@ -137,43 +91,6 @@ const collections = (connection: Connection, dbVersionSupport: DbVersionSupport) moduleConfig[config.reranker.name] = config.reranker.config ? config.reranker.config : {}; } - const makeVectorsConfig = (configVectorizers: VectorizersConfigCreate) => { - let vectorizers: string[] = []; - const vectorsConfig: Record = {}; - const vectorizersConfig = Array.isArray(configVectorizers) - ? configVectorizers - : [ - { - ...configVectorizers, - name: 'default', - }, - ]; - vectorizersConfig.forEach((v) => { - if (v.vectorIndex.name === 'dynamic' && !supportsDynamicVectorIndex.supports) { - throw new WeaviateUnsupportedFeatureError(supportsDynamicVectorIndex.message); - } - const vectorConfig: any = { - vectorIndexConfig: parseVectorIndex(v.vectorIndex), - vectorIndexType: v.vectorIndex.name, - vectorizer: {}, - }; - const vectorizer = - v.vectorizer.name === 'text2vec-azure-openai' ? 'text2vec-openai' : v.vectorizer.name; - vectorizers = [...vectorizers, vectorizer]; - vectorConfig.vectorizer[vectorizer] = { - properties: v.properties, - ...parseVectorizerConfig(v.vectorizer.config), - }; - if (v.name === undefined) { - throw new WeaviateInvalidInputError( - 'vectorName is required for each vectorizer when specifying more than one vectorizer' - ); - } - vectorsConfig[v.name] = vectorConfig; - }); - return { vectorsConfig, vectorizers }; - }; - const makeLegacyVectorizer = ( configVectorizers: VectorConfigCreate, undefined, string, Vectorizer> ) => { @@ -221,7 +138,7 @@ const collections = (connection: Connection, dbVersionSupport: DbVersionSupport) let vectorizers: string[] = []; if (supportsNamedVectors.supports) { const { vectorsConfig, vectorizers: vecs } = config.vectorizers - ? makeVectorsConfig(config.vectorizers) + ? makeVectorsConfig(config.vectorizers, supportsDynamicVectorIndex) : { vectorsConfig: undefined, vectorizers: [] }; schema.vectorConfig = vectorsConfig; vectorizers = [...vecs]; @@ -230,7 +147,7 @@ const collections = (connection: Connection, dbVersionSupport: DbVersionSupport) throw new WeaviateUnsupportedFeatureError(supportsNamedVectors.message); } const configs = config.vectorizers - ? makeLegacyVectorizer(config.vectorizers) + ? makeLegacyVectorizer({ ...config.vectorizers, name: undefined }) : { vectorizer: undefined, moduleConfig: undefined, diff --git a/src/collections/integration.test.ts b/src/collections/integration.test.ts index f143843d..df424745 100644 --- a/src/collections/integration.test.ts +++ b/src/collections/integration.test.ts @@ -174,6 +174,20 @@ describe('Testing of the collections.create method', () => { expect(response.vectorizers.default.vectorizer.name).toEqual('text2vec-contextionary'); }); + it('should be able to create a collection with 1 custom named vector', async () => { + const collectionName = 'TestCollectionSingleCustomNamedVector'; + const response = await contextionary.collections + .create({ + name: collectionName, + vectorizers: weaviate.configure.vectorizer.none({ name: 'custom' }), + }) + .then(() => contextionary.collections.use(collectionName).config.get()); + expect(response.name).toEqual(collectionName); + expect(response.properties?.length).toEqual(0); + expect(response.vectorizers.custom.indexConfig).toBeDefined(); + expect(response.vectorizers.custom.indexType).toEqual('hnsw'); + }); + it('should be able to create a simple collection without a generic using a schema var', async () => { const collectionName = 'TestCollectionSimpleNonGenericVar'; const schema = { diff --git a/src/schema/vectorAdder.ts b/src/schema/vectorAdder.ts new file mode 100644 index 00000000..ccdd494e --- /dev/null +++ b/src/schema/vectorAdder.ts @@ -0,0 +1,59 @@ +import Connection from '../connection/index.js'; +import { VectorConfig } from '../index.js'; +import { CommandBase } from '../validation/commandBase.js'; +import { isValidStringProperty } from '../validation/string.js'; +import ClassGetter from './classGetter.js'; + +export default class VectorAdder extends CommandBase { + private className!: string; + private vectors!: Record; + + constructor(client: Connection) { + super(client); + } + + withClassName = (className: string) => { + this.className = className; + return this; + }; + withVectors = (vectors: Record) => { + this.vectors = vectors; + return this; + }; + + validateClassName = () => { + if (!isValidStringProperty(this.className)) { + this.addError('className must be set - set with .withClassName(className)'); + } + }; + + validate = () => { + this.validateClassName(); + }; + + do = (): Promise => { + this.validate(); + if (this.errors.length > 0) { + return Promise.reject(new Error('invalid usage: ' + this.errors.join(', '))); + } + + return new ClassGetter(this.client) + .withClassName(this.className) + .do() + .then(async (schema) => { + if (schema.vectorConfig === undefined) { + schema.vectorConfig = {}; + } + + for (const [key, value] of Object.entries(this.vectors)) { + if (schema.vectorConfig[key] !== undefined) { + continue; + } + schema.vectorConfig![key] = { ...value }; + } + + const path = `/schema/${this.className}`; + await this.client.put(path, schema); + }); + }; +}