diff --git a/README.md b/README.md index 31d0e3cc333..e7d92f76640 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,7 @@ AnythingLLM divides your documents into objects called `workspaces`. A Workspace - [LanceDB](https://github.com/lancedb/lancedb) (default) - [PGVector](https://github.com/pgvector/pgvector) +- [Vastbase](https://www.vastbase.com) - [Astra DB](https://www.datastax.com/products/datastax-astra) - [Pinecone](https://pinecone.io) - [Chroma & ChromaCloud](https://trychroma.com) diff --git a/docker/.env.example b/docker/.env.example index 27fa1c013c6..4cdecd43ada 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -218,6 +218,11 @@ GID='1000' # PGVECTOR_CONNECTION_STRING="postgresql://dbuser:dbuserpass@localhost:5432/yourdb" # PGVECTOR_TABLE_NAME="anythingllm_vectors" # optional, but can be defined +# Enable all below if you are using vector database: Vastbase. +# VECTOR_DB="vastbase" +# VASTBASE_CONNECTION_STRING="postgresql://dbuser:dbuserpass@localhost:5432/yourdb" +# VASTBASE_TABLE_NAME="anythingllm_vectors" # optional, but can be defined + # Enable all below if you are using vector database: Chroma. # VECTOR_DB="chroma" # CHROMA_ENDPOINT='http://host.docker.internal:8000' diff --git a/frontend/src/components/VectorDBSelection/VastbaseOptions/index.jsx b/frontend/src/components/VectorDBSelection/VastbaseOptions/index.jsx new file mode 100644 index 00000000000..9bef1fca07d --- /dev/null +++ b/frontend/src/components/VectorDBSelection/VastbaseOptions/index.jsx @@ -0,0 +1,103 @@ +import { Info } from "@phosphor-icons/react"; +import { Tooltip } from "react-tooltip"; + +export default function VastbaseOptions({ settings }) { + return ( +
+
+
+
+ + + +

+ This is the connection string for the Postgres database in the + format of
+ postgresql://username:password@host:port/database +
+
+ The user for the database must have the following permissions: +

    +
  • Read access to the database
    • +
  • Read access to the database schema
    • +
  • Create access to the database
    • +
+
+ + You must have the vastbase extension installed on the + database. + +

+ +
+ +
+ +
+
+ + + +

+ This is the name of the table in the Postgres database that will + store the vectors. +
+
+ By default, the table name is anythingllm_vectors. +
+
+ + This table must not already exist on the database - it will be + created automatically. + +

+ +
+ +
+
+
+ ); +} diff --git a/frontend/src/media/vectordbs/vastbase.png b/frontend/src/media/vectordbs/vastbase.png new file mode 100644 index 00000000000..9d5ab1a0b35 Binary files /dev/null and b/frontend/src/media/vectordbs/vastbase.png differ diff --git a/frontend/src/pages/GeneralSettings/VectorDatabase/index.jsx b/frontend/src/pages/GeneralSettings/VectorDatabase/index.jsx index 864e5533ff8..e4994ef0ebd 100644 --- a/frontend/src/pages/GeneralSettings/VectorDatabase/index.jsx +++ b/frontend/src/pages/GeneralSettings/VectorDatabase/index.jsx @@ -21,6 +21,7 @@ import MilvusLogo from "@/media/vectordbs/milvus.png"; import ZillizLogo from "@/media/vectordbs/zilliz.png"; import AstraDBLogo from "@/media/vectordbs/astraDB.png"; import PGVectorLogo from "@/media/vectordbs/pgvector.png"; +import VastbaseLogo from "@/media/vectordbs/vastbase.png"; import LanceDBOptions from "@/components/VectorDBSelection/LanceDBOptions"; import ChromaDBOptions from "@/components/VectorDBSelection/ChromaDBOptions"; @@ -32,6 +33,7 @@ import MilvusDBOptions from "@/components/VectorDBSelection/MilvusDBOptions"; import ZillizCloudOptions from "@/components/VectorDBSelection/ZillizCloudOptions"; import AstraDBOptions from "@/components/VectorDBSelection/AstraDBOptions"; import PGVectorOptions from "@/components/VectorDBSelection/PGVectorOptions"; +import VastbaseOptions from "@/components/VectorDBSelection/VastbaseOptions"; export default function GeneralVectorDatabase() { const [saving, setSaving] = useState(false); @@ -126,6 +128,13 @@ export default function GeneralVectorDatabase() { options: , description: "Vector search powered by PostgreSQL.", }, + { + name: "Vastbase", + value: "vastbase", + logo: VastbaseLogo, + options: , + description: "Vector search powered by Vastbase.", + }, { name: "Chroma", value: "chroma", diff --git a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx index b12979a889d..ea618c98309 100644 --- a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx +++ b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx @@ -37,6 +37,7 @@ import MilvusLogo from "@/media/vectordbs/milvus.png"; import VoyageAiLogo from "@/media/embeddingprovider/voyageai.png"; import PPIOLogo from "@/media/llmprovider/ppio.png"; import PGVectorLogo from "@/media/vectordbs/pgvector.png"; +import VastbaseLogo from "@/media/vectordbs/vastbase.png"; import DPAISLogo from "@/media/llmprovider/dpais.png"; import MoonshotAiLogo from "@/media/llmprovider/moonshotai.png"; import CometApiLogo from "@/media/llmprovider/cometapi.png"; @@ -280,6 +281,14 @@ export const VECTOR_DB_PRIVACY = { ], logo: PGVectorLogo, }, + vastbase: { + name: "Vastbase", + description: [ + "Your vectors and document text are stored on your Vastbase instance", + "Access to your instance is managed by you", + ], + logo: VastbaseLogo, + }, chroma: { name: "Chroma", description: [ diff --git a/locales/README.fa-IR.md b/locales/README.fa-IR.md index c28abf648ae..fe57af78d4c 100644 --- a/locales/README.fa-IR.md +++ b/locales/README.fa-IR.md @@ -138,6 +138,7 @@ AnythingLLM اسناد شما را به اشیایی به نام `workspaces` ت - [LanceDB](https://github.com/lancedb/lancedb) (پیش‌فرض) - [PGVector](https://github.com/pgvector/pgvector) +- [Vastbase](https://www.vastdata.com.cn) - [Astra DB](https://www.datastax.com/products/datastax-astra) - [Pinecone](https://pinecone.io) - [Chroma](https://trychroma.com) diff --git a/locales/README.ja-JP.md b/locales/README.ja-JP.md index d6fef0fa5b7..a7dba16d59f 100644 --- a/locales/README.ja-JP.md +++ b/locales/README.ja-JP.md @@ -122,6 +122,7 @@ AnythingLLMは、ドキュメントを`ワークスペース`と呼ばれるオ - [LanceDB](https://github.com/lancedb/lancedb)（デフォルト） - [PGVector](https://github.com/pgvector/pgvector) +- [Vastbase](https://www.vastdata.com.cn) - [Astra DB](https://www.datastax.com/products/datastax-astra) - [Pinecone](https://pinecone.io) - [Chroma](https://trychroma.com) diff --git a/locales/README.tr-TR.md b/locales/README.tr-TR.md index 9f539779503..c3cdacdc34e 100644 --- a/locales/README.tr-TR.md +++ b/locales/README.tr-TR.md @@ -133,6 +133,7 @@ AnythingLLM, belgelerinizi **"çalışma alanları" (workspaces)** adı verilen - [LanceDB](https://github.com/lancedb/lancedb) (default) - [PGVector](https://github.com/pgvector/pgvector) +- [Vastbase](https://www.vastdata.com.cn) - [Astra DB](https://www.datastax.com/products/datastax-astra) - [Pinecone](https://pinecone.io) - [Chroma](https://trychroma.com) diff --git a/locales/README.zh-CN.md b/locales/README.zh-CN.md index aa328351449..24a73b2848a 100644 --- a/locales/README.zh-CN.md +++ b/locales/README.zh-CN.md @@ -133,6 +133,7 @@ AnythingLLM将您的文档划分为称为`workspaces` (工作区)的对象。工 - [LanceDB](https://github.com/lancedb/lancedb) （默认） - [PGVector](https://github.com/pgvector/pgvector) +- [Vastbase](https://www.vastdata.com.cn) - [Astra DB](https://www.datastax.com/products/datastax-astra) - [Pinecone](https://pinecone.io) - [Chroma](https://trychroma.com) diff --git a/server/.env.example b/server/.env.example index 3dc0bd596c4..426d73b4e12 100644 --- a/server/.env.example +++ b/server/.env.example @@ -239,6 +239,11 @@ VECTOR_DB="lancedb" # PGVECTOR_CONNECTION_STRING="postgresql://dbuser:dbuserpass@localhost:5432/yourdb" # PGVECTOR_TABLE_NAME="anythingllm_vectors" # optional, but can be defined +# Enable all below if you are using vector database: Vastbase. +# VECTOR_DB="vastbase" +# VASTBASE_CONNECTION_STRING="postgresql://dbuser:dbuserpass@localhost:5432/yourdb" +# VASTBASE_TABLE_NAME="anythingllm_vectors" # optional, but can be defined + # Enable all below if you are using vector database: Weaviate. # VECTOR_DB="weaviate" # WEAVIATE_ENDPOINT="http://localhost:8080" diff --git a/server/__tests__/utils/vectorDbProviders/vastbase/index.test.js b/server/__tests__/utils/vectorDbProviders/vastbase/index.test.js new file mode 100644 index 00000000000..ff11dd55802 --- /dev/null +++ b/server/__tests__/utils/vectorDbProviders/vastbase/index.test.js @@ -0,0 +1,76 @@ +const { Vastbase } = require("../../../../utils/vectorDbProviders/vastbase"); + +describe("Vastbase.sanitizeForJsonb", () => { + it("returns null/undefined as-is", () => { + expect(Vastbase.sanitizeForJsonb(null)).toBeNull(); + expect(Vastbase.sanitizeForJsonb(undefined)).toBeUndefined(); + }); + + it("keeps safe whitespace (tab, LF, CR) and removes disallowed C0 controls", () => { + const input = "a\u0000\u0001\u0002\tline\ncarriage\rreturn\u001Fend"; + const result = Vastbase.sanitizeForJsonb(input); + // Expect all < 0x20 except 9,10,13 removed; keep letters and allowed whitespace + expect(result).toBe("a\tline\ncarriage\rreturnend"); + }); + + it("removes only disallowed control chars; keeps normal printable chars", () => { + const input = "Hello\u0000, World! \u0007\u0008\u000B\u000C\u001F"; + const result = Vastbase.sanitizeForJsonb(input); + expect(result).toBe("Hello, World! "); + }); + + it("deeply sanitizes objects", () => { + const input = { + plain: "ok", + bad: "has\u0000nul", + nested: { + arr: ["fine", "bad\u0001", { deep: "\u0002oops" }], + }, + }; + const result = Vastbase.sanitizeForJsonb(input); + expect(result).toEqual({ + plain: "ok", + bad: "hasnul", + nested: { arr: ["fine", "bad", { deep: "oops" }] }, + }); + }); + + it("deeply sanitizes arrays", () => { + const input = ["\u0000", 1, true, { s: "bad\u0003" }, ["ok", "\u0004bad"]]; + const result = Vastbase.sanitizeForJsonb(input); + expect(result).toEqual(["", 1, true, { s: "bad" }, ["ok", "bad"]]); + }); + + it("converts Date to ISO string", () => { + const d = new Date("2020-01-02T03:04:05.000Z"); + expect(Vastbase.sanitizeForJsonb(d)).toBe(d.toISOString()); + }); + + it("returns primitives unchanged (number, boolean, bigint)", () => { + expect(Vastbase.sanitizeForJsonb(42)).toBe(42); + expect(Vastbase.sanitizeForJsonb(3.14)).toBe(3.14); + expect(Vastbase.sanitizeForJsonb(true)).toBe(true); + expect(Vastbase.sanitizeForJsonb(false)).toBe(false); + expect(Vastbase.sanitizeForJsonb(BigInt(1))).toBe(BigInt(1)); + }); + + it("returns symbol unchanged", () => { + const sym = Symbol("x"); + expect(Vastbase.sanitizeForJsonb(sym)).toBe(sym); + }); + + it("does not mutate original objects/arrays", () => { + const obj = { a: "bad\u0000", nested: { b: "ok" } }; + const arr = ["\u0001", { c: "bad\u0002" }]; + const objCopy = JSON.parse(JSON.stringify(obj)); + const arrCopy = JSON.parse(JSON.stringify(arr)); + const resultObj = Vastbase.sanitizeForJsonb(obj); + const resultArr = Vastbase.sanitizeForJsonb(arr); + // Original inputs remain unchanged + expect(obj).toEqual(objCopy); + expect(arr).toEqual(arrCopy); + // Results are sanitized copies + expect(resultObj).toEqual({ a: "bad", nested: { b: "ok" } }); + expect(resultArr).toEqual(["", { c: "bad" }]); + }); +}); diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index 3a7a4b21554..484f79a9b85 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -8,6 +8,7 @@ const prisma = require("../utils/prisma"); const { v4 } = require("uuid"); const { MetaGenerator } = require("../utils/boot/MetaGenerator"); const { PGVector } = require("../utils/vectorDbProviders/pgvector"); +const { Vastbase } = require("../utils/vectorDbProviders/vastbase"); const { NativeEmbedder } = require("../utils/EmbeddingEngines/native"); const { getBaseLLMProviderModel } = require("../utils/helpers"); @@ -458,6 +459,10 @@ const SystemSettings = { // PGVector Keys PGVectorConnectionString: !!PGVector.connectionString() || false, PGVectorTableName: PGVector.tableName(), + + // Vastbase Keys + VastbaseConnectionString: !!Vastbase.connectionString() || false, + VastbaseTableName: Vastbase.tableName(), }; }, diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index 819a464c6d0..da603cbff06 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -113,6 +113,9 @@ function getVectorDbClass(getExactly = null) { case "pgvector": const { PGVector } = require("../vectorDbProviders/pgvector"); return PGVector; + case "vastbase": + const { Vastbase } = require("../vectorDbProviders/vastbase"); + return Vastbase; default: throw new Error("ENV: No VECTOR_DB value found in environment!"); } diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index c8109efb193..447e91d5128 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -425,6 +425,23 @@ const KEY_MAPPING = { preUpdate: [validatePGVectorTableName], }, + /* + Vastbase Options + - Does very simple validations - we should expand this in the future + - to ensure the connection string is valid and the table name is valid + - via direct query + */ + VastbaseConnectionString: { + envKey: "VASTBASE_CONNECTION_STRING", + checks: [isNotEmpty, looksLikePostgresConnectionString], + preUpdate: [validateVastbaseConnectionString], + }, + VastbaseTableName: { + envKey: "VASTBASE_TABLE_NAME", + checks: [isNotEmpty], + preUpdate: [validateVastbaseTableName], + }, + // Together Ai Options TogetherAiApiKey: { envKey: "TOGETHER_AI_API_KEY", @@ -906,6 +923,7 @@ function supportedVectorDB(input = "") { "zilliz", "astra", "pgvector", + "vastbase", ]; return supported.includes(input) ? null @@ -1064,6 +1082,70 @@ async function validatePGVectorTableName(key, prevValue, nextValue) { return null; } +/** + * Validates the Postgres connection string for the PGVector options. + * @param {string} input - The Postgres connection string to validate. + * @returns {string} - An error message if the connection string is invalid, otherwise null. + */ +async function looksLikeVastbaseConnectionString(connectionString = null) { + if (!connectionString || !connectionString.startsWith("postgresql://")) + return "Invalid Vastbase connection string. Must start with postgresql://"; + if (connectionString.includes(" ")) + return "Invalid Vastbase connection string. Must not contain spaces."; + return null; +} + +/** + * Validates the Postgres connection string for the PGVector options. + * @param {string} key - The ENV key we are validating. + * @param {string} prevValue - The previous value of the key. + * @param {string} nextValue - The next value of the key. + * @returns {string} - An error message if the connection string is invalid, otherwise null. + */ +async function validateVastbaseConnectionString(key, prevValue, nextValue) { + const envKey = KEY_MAPPING[key].envKey; + + if (prevValue === nextValue) return; // If the value is the same as the previous value, don't validate it. + if (!nextValue) return; // If the value is not set, don't validate it. + if (nextValue === process.env[envKey]) return; // If the value is the same as the current connection string, don't validate it. + + const { Vastbase } = require("../vectorDbProviders/vastbase"); + const { error, success } = await Vastbase.validateConnection({ + connectionString: nextValue, + }); + if (!success) return error; + + // Set the ENV variable for the Vastbase connection string early so we can use it in the table check. + process.env[envKey] = nextValue; + return null; +} + +/** + * Validates the Postgres table name for the Vastbase options. + * - Table should not already exist in the database. + * @param {string} key - The ENV key we are validating. + * @param {string} prevValue - The previous value of the key. + * @param {string} nextValue - The next value of the key. + * @returns {string} - An error message if the table name is invalid, otherwise null. + */ +async function validateVastbaseTableName(key, prevValue, nextValue) { + const envKey = KEY_MAPPING[key].envKey; + + if (prevValue === nextValue) return; // If the value is the same as the previous value, don't validate it. + if (!nextValue) return; // If the value is not set, don't validate it. + if (nextValue === process.env[envKey]) return; // If the value is the same as the current table name, don't validate it. + if (!process.env.VASTBASE_CONNECTION_STRING) return; // if connection string is not set, don't validate it since it will fail. + + const { Vastbase } = require("../vectorDbProviders/vastbase"); + const { error, success } = await Vastbase.validateConnection({ + connectionString: process.env.VASTBASE_CONNECTION_STRING, + tableName: nextValue, + }); + if (!success) return error; + + return null; +} + // This will force update .env variables which for any which reason were not able to be parsed or // read from an ENV file as this seems to be a complicating step for many so allowing people to write // to the process will at least alleviate that issue. It does not perform comprehensive validity checks or sanity checks diff --git a/server/utils/vectorDbProviders/vastbase/SETUP.md b/server/utils/vectorDbProviders/vastbase/SETUP.md new file mode 100644 index 00000000000..f312bece7a9 --- /dev/null +++ b/server/utils/vectorDbProviders/vastbase/SETUP.md @@ -0,0 +1,119 @@ +# Setting up `Vastbase` for AnythingLLM + +Setting up Vastbase for anythingllm to use as your vector database is quite easy. At a minimum, you will need the following: + +- Vastbase v3.0.8+ +- [`vastbase`](https://docs.vastdata.com.cn/zh_CN/VastbaseG100/V3.0.8/1) extension installed on DB +- User with DB table creation perms and READ access + +## Setup on CentOS7.9 (example) + +### Install Vastbase DB +1. install vastbase refer to [Vastbase Installation Guide](https://docs.vastdata.com.cn/zh_CN/VastbaseG100/V3.0.8/1) +2. connect to the db as superuser and create the extension + ```bash + vsql -r -d + ``` + +### Set Vastbase as your vector db + +_this can be done via the UI or by directly editing the `.env` file_ + +First, obtain a valid connection string for the user, credentials, and db you want to target. +eg: `postgresql://dbuser:dbuserpass@localhost:5432/yourdb` + +> [!IMPORTANT] +> If you have an existing table that you want to use as a vector database, AnythingLLM **requires** that the table be +> at least minimally conform to the expected schema - this can be seen in the [index.js](./index.js) file. + +_optional_ - set a table name you wish to have AnythingLLM store vectors to. By default this is `anythingllm_vectors` + +## Common Questions + +### I cannot connect to the DB (Running AnythingLLM in Docker) + +If you are running AnythingLLM in Docker, you will need to ensure that the DB is accessible from the container. +If you are running your DB in another Docker container **or** on the host machine, you will need to ensure that the container can access the DB. + +`localhost` will not work in this case as it will attempt to connect to the DB _inside the AnythingLLM container_ instead of the host machine or another container. + +You will need to use the `host.docker.internal` (or `172.17.0.1` on Linux/Ubuntu) address. + +``` +on Mac or Windows: +postgresql://dbuser:dbuserpass@localhost:5432/yourdb => postgresql://dbuser:dbuserpass@host.docker.internal:5432/yourdb + +on Linux: +postgresql://dbuser:dbuserpass@localhost:5432/yourdb => postgresql://dbuser:dbuserpass@172.17.0.1:5432/yourdb +``` + +### Can I use an existing table as a vector database? + +Yes, you can use an existing table as a vector database. However, AnythingLLM **requires** that the table be at least minimally conform to the expected schema - this can be seen in the [index.js](./index.js) file. + +It is **absolutely critical** that the `embedding` column's `VECTOR(XXXX)` dimensions match the dimension of the embedder in AnythingLLM. The default embedding model is 384 dimensions. However, if you are using a custom embedder, you will need to ensure that the dimension value is set correctly. + +### Validate the connection to the database + +When setting the connection string in or table name via the AnythingLLM UI, the following validations will be attempted: + +- Validate the connection string +- Validate the table name +- Run test connection to ensure the table exists and is accessible by the connection string used +- Check if the table name already exists and if so, validate that it is an embedding table with the correct schema + +### My embedding table is not present in the DB + +The embedding storage table is created by AnythingLLM **on the first upsert** of a vector. If you have not yet embedding any documents, the table will not be present in the DB. + +### How do I reset my vector database? + +_at the workspace level in Settings > Vector Database_ + +You can use the "Reset Vector Database" button in the AnythingLLM UI to reset your vector database. This will drop all vectors within that workspace, but the table will remain in the DB. + +_reset the vector database at the db level_ + +For this, you will need to `DROP TABLE` from the command line or however you manage your DB. Once the table is dropped, it will be recreated by AnythingLLM on the next upsert. + +## Troubleshooting + +### Cannot connect to DB + +- Ensure the connection string is valid +- Ensure the user has access to the database +- Ensure the vastbase extension is installed + +### Cannot create table + +- Ensure the user has `CREATE TABLE` permissions + +### Cannot insert vector + +- Ensure the user has `INSERT` permissions in the database +- Ensure the table has a dimension value set and this matches the dimension of the embedder in AnythingLLM +- Ensure the table has a vector column set + +### Cannot query vector + +- Ensure the user has `SELECT` permissions in the database +- Ensure the table has a vector column set +- Ensure the table has a dimension value set and this matches the dimension of the embedder in AnythingLLM + +### "type 'floatvector' does not exist" issues with Vastbase + +If you are using the Vastbase as your vector database, you may encounter an error similar to the following when embedding documents: + +``` +type 'floatvector' does not exist +``` + +This is due to the fact that the vastbase version is less than v3.0.8. + +First, follow the instructions in the [Vastbase README](https://docs.vastdata.com.cn/zh_CN/VastbaseG100/V3.0.8/1/) to install v3.0.8 version. + +Then, you will need to create the extension on the database. This can be done by running the following command: + +```bash +vsql -r -d +``` diff --git a/server/utils/vectorDbProviders/vastbase/index.js b/server/utils/vectorDbProviders/vastbase/index.js new file mode 100644 index 00000000000..0723682573d --- /dev/null +++ b/server/utils/vectorDbProviders/vastbase/index.js @@ -0,0 +1,838 @@ +const vsql = require("pg"); +const { toChunks, getEmbeddingEngineSelection } = require("../../helpers"); +const { TextSplitter } = require("../../TextSplitter"); +const { v4: uuidv4 } = require("uuid"); +const { sourceIdentifier } = require("../../chats"); + +/* + Embedding Table Schema (table name defined by user) + - id: UUID PRIMARY KEY + - namespace: TEXT + - embedding: floatvector(xxxx) + - metadata: JSONB + - created_at: TIMESTAMP +*/ + +const Vastbase = { + name: "Vastbase", + connectionTimeout: 30_000, + /** + * Get the table name for the Vastbase database. + * - Defaults to "anythingllm_vectors" if no table name is provided. + * @returns {string} + */ + tableName: () => process.env.VASTBASE_TABLE_NAME || "anythingllm_vectors", + + /** + * Get the connection string for the Vastbase database. + * - Requires a connection string to be present in the environment variables. + * @returns {string | null} + */ + connectionString: () => process.env.VASTBASE_CONNECTION_STRING, + + // Possible for this to be a user-configurable option in the future. + // Will require a handler per operator to ensure scores are normalized. + operator: { + l2: "<->", + innerProduct: "<#>", + cosine: "<=>", + l1: "<+>", + hamming: "<~>", + jaccard: "<%>", + }, + getTablesSql: + "SELECT * FROM pg_catalog.pg_tables WHERE schemaname = current_schema()", + getEmbeddingTableSchemaSql: + "SELECT column_name,data_type FROM information_schema.columns WHERE table_name = $1", + createTableSql: (dimensions) => + `CREATE TABLE IF NOT EXISTS "${Vastbase.tableName()}" (id UUID PRIMARY KEY, namespace TEXT, embedding floatvector(${Number(dimensions)}), metadata JSONB, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP)`, + + log: function (message = null, ...args) { + + console.log(`\x1b[35m[Vastbase]\x1b[0m ${message}`, ...args); + }, + + /** + * Recursively sanitize values intended for JSONB to prevent Postgres errors + * like "unsupported Unicode escape sequence". This primarily removes the + * NUL character (\u0000) and other disallowed control characters from + * strings. Arrays and objects are traversed and sanitized deeply. + * @param {any} value + * @returns {any} + */ + sanitizeForJsonb: function (value) { + // Fast path for null/undefined and primitives that do not need changes + if (value === null || value === undefined) return value; + + // Strings: strip NUL and unsafe C0 control characters except common whitespace + if (typeof value === "string") { + // Build a sanitized string by excluding C0 control characters except + // horizontal tab (9), line feed (10), and carriage return (13). + let sanitized = ""; + for (let i = 0; i < value.length; i++) { + const code = value.charCodeAt(i); + if (code === 9 || code === 10 || code === 13 || code >= 0x20) { + sanitized += value[i]; + } + } + return sanitized; + } + + // Arrays: sanitize each element + if (Array.isArray(value)) { + return value.map((item) => this.sanitizeForJsonb(item)); + } + + // Dates: keep as ISO string + if (value instanceof Date) { + return value.toISOString(); + } + + // Objects: sanitize each property value + if (typeof value === "object") { + const result = {}; + for (const [k, v] of Object.entries(value)) { + result[k] = this.sanitizeForJsonb(v); + } + return result; + } + + // Numbers, booleans, etc. + return value; + }, + + client: function (connectionString = null) { + return new vsql.Client({ + connectionString: connectionString || Vastbase.connectionString(), + }); + }, + + /** + * Validate the existing embedding table schema. + * @param {vsql.Client} pgClient + * @param {string} tableName + * @returns {Promise} + */ + validateExistingEmbeddingTableSchema: async function (pgClient, tableName) { + const result = await pgClient.query(this.getEmbeddingTableSchemaSql, [ + tableName, + ]); + + // Minimum expected schema for an embedding table. + // Extra columns are allowed but the minimum exact columns are required + // to be present in the table. + const expectedSchema = [ + { + column_name: "id", + expected: "uuid", + validation: function (dataType) { + return dataType.toLowerCase() === this.expected; + }, + }, + { + column_name: "namespace", + expected: "text", + validation: function (dataType) { + return dataType.toLowerCase() === this.expected; + }, + }, + { + column_name: "embedding", + expected: "floatvector", + validation: function (dataType) { + return !!dataType; + }, + }, // just check if it exists + { + column_name: "metadata", + expected: "jsonb", + validation: function (dataType) { + return dataType.toLowerCase() === this.expected; + }, + }, + { + column_name: "created_at", + expected: "timestamp", + validation: function (dataType) { + return dataType.toLowerCase().includes(this.expected); + }, + }, + ]; + + if (result.rows.length === 0) + throw new Error( + `The table '${tableName}' was found but does not contain any columns or cannot be accessed by role. It cannot be used as an embedding table in AnythingLLM.` + ); + + for (const rowDef of expectedSchema) { + const column = result.rows.find( + (c) => c.column_name === rowDef.column_name + ); + if (!column) + throw new Error( + `The column '${rowDef.column_name}' was expected but not found in the table '${tableName}'.` + ); + if (!rowDef.validation(column.data_type)) + throw new Error( + `Invalid data type for column: '${column.column_name}'. Got '${column.data_type}' but expected '${rowDef.expected}'` + ); + } + + this.log( + `✅ The vastbase table '${tableName}' was found and meets the minimum expected schema for an embedding table.` + ); + return true; + }, + + /** + * Validate the connection to the database and verify that the table does not already exist. + * so that anythingllm can manage the table directly. + * + * @param {{connectionString: string | null, tableName: string | null}} params + * @returns {Promise<{error: string | null, success: boolean}>} + */ + validateConnection: async function ({ + connectionString = null, + tableName = null, + }) { + if (!connectionString) throw new Error("No connection string provided"); + + try { + const timeoutPromise = new Promise((resolve) => { + setTimeout(() => { + resolve({ + error: `Connection timeout (${(Vastbase.connectionTimeout / 1000).toFixed(0)}s). Please check your connection string and try again.`, + success: false, + }); + }, Vastbase.connectionTimeout); + }); + + const connectionPromise = new Promise(async (resolve) => { + let pgClient = null; + try { + pgClient = this.client(connectionString); + await pgClient.connect(); + const result = await pgClient.query(this.getTablesSql); + + if (result.rows.length !== 0 && !!tableName) { + const tableExists = result.rows.some( + (row) => row.tablename === tableName + ); + if (tableExists) + await this.validateExistingEmbeddingTableSchema( + pgClient, + tableName + ); + } + resolve({ error: null, success: true }); + } catch (err) { + resolve({ error: err.message, success: false }); + } finally { + if (pgClient) await pgClient.end(); + } + }); + + // Race the connection attempt against the timeout + const result = await Promise.race([connectionPromise, timeoutPromise]); + return result; + } catch (err) { + this.log("Validation Error:", err.message); + let readableError = err.message; + switch (true) { + case err.message.includes("ECONNREFUSED"): + readableError = + "The host could not be reached. Please check your connection string and try again."; + break; + default: + break; + } + return { error: readableError, success: false }; + } + }, + + /** + * Test the connection to the database directly. + * @returns {{error: string | null, success: boolean}} + */ + testConnectionToDB: async function () { + try { + const pgClient = await this.connect(); + await pgClient.query(this.getTablesSql); + await pgClient.end(); + return { error: null, success: true }; + } catch (err) { + return { error: err.message, success: false }; + } + }, + + /** + * Connect to the database. + * - Throws an error if the connection string or table name is not provided. + * @returns {Promise} + */ + connect: async function () { + if (!Vastbase.connectionString()) + throw new Error("No connection string provided"); + if (!Vastbase.tableName()) throw new Error("No table name provided"); + + const client = this.client(); + await client.connect(); + return client; + }, + + /** + * Test the connection to the database with already set credentials via ENV + * @returns {{error: string | null, success: boolean}} + */ + heartbeat: async function () { + return this.testConnectionToDB(); + }, + + /** + * Check if the anythingllm embedding table exists in the database + * @returns {Promise} + */ + dbTableExists: async function () { + let connection = null; + try { + connection = await this.connect(); + const tables = await connection.query(this.getTablesSql); + if (tables.rows.length === 0) return false; + const tableExists = tables.rows.some( + (row) => row.tablename === Vastbase.tableName() + ); + return !!tableExists; + } catch (err) { + return false; + } finally { + if (connection) await connection.end(); + } + }, + + totalVectors: async function () { + if (!(await this.dbTableExists())) return 0; + let connection = null; + try { + connection = await this.connect(); + const result = await connection.query( + `SELECT COUNT(id) FROM "${Vastbase.tableName()}"` + ); + return result.rows[0].count; + } catch (err) { + return 0; + } finally { + if (connection) await connection.end(); + } + }, + + // Distance for cosine is just the distance for floatvector. + distanceToSimilarity: function (distance = null) { + if (distance === null || typeof distance !== "number") return 0.0; + if (distance >= 1.0) return 1; + if (distance < 0) return 1 - Math.abs(distance); + return 1 - distance; + }, + + namespaceCount: async function (namespace = null) { + if (!(await this.dbTableExists())) return 0; + let connection = null; + try { + connection = await this.connect(); + const result = await connection.query( + `SELECT COUNT(id) FROM "${Vastbase.tableName()}" WHERE namespace = $1`, + [namespace] + ); + return result.rows[0].count; + } catch (err) { + return 0; + } finally { + if (connection) await connection.end(); + } + }, + + /** + * Performs a SimilaritySearch on a given Vastbase namespace. + * @param {Object} params + * @param {vsql.Client} params.client + * @param {string} params.namespace + * @param {number[]} params.queryVector + * @param {number} params.similarityThreshold + * @param {number} params.topN + * @param {string[]} params.filterIdentifiers + * @returns + */ + similarityResponse: async function ({ + client, + namespace, + queryVector, + similarityThreshold = 0.25, + topN = 4, + filterIdentifiers = [], + }) { + const result = { + contextTexts: [], + sourceDocuments: [], + scores: [], + }; + + const embedding = `[${queryVector.map(Number).join(",")}]`; + const response = await client.query( + `SELECT embedding ${this.operator.cosine} $1 AS _distance, metadata FROM "${Vastbase.tableName()}" WHERE namespace = $2 ORDER BY _distance ASC LIMIT $3`, + [embedding, namespace, topN] + ); + response.rows.forEach((item) => { + if (this.distanceToSimilarity(item._distance) < similarityThreshold) + return; + if (filterIdentifiers.includes(sourceIdentifier(item.metadata))) { + this.log( + "A source was filtered from context as it's parent document is pinned." + ); + return; + } + + result.contextTexts.push(item.metadata.text); + result.sourceDocuments.push({ + ...item.metadata, + score: this.distanceToSimilarity(item._distance), + }); + result.scores.push(this.distanceToSimilarity(item._distance)); + }); + + return result; + }, + + normalizeVector: function (vector) { + const magnitude = Math.sqrt( + vector.reduce((sum, val) => sum + val * val, 0) + ); + if (magnitude === 0) return vector; // Avoid division by zero + return vector.map((val) => val / magnitude); + }, + + /** + * Update or create a collection in the database + * @param {Object} params + * @param {vsql.Connection} params.connection + * @param {{id: number, vector: number[], metadata: Object}[]} params.submissions + * @param {string} params.namespace + * @param {number} params.dimensions + * @returns {Promise} + */ + updateOrCreateCollection: async function ({ + connection, + submissions, + namespace, + dimensions = 384, + }) { + await this.createTableIfNotExists(connection, dimensions); + this.log(`Updating or creating collection ${namespace}`); + + try { + // Create a transaction of all inserts + await connection.query(`BEGIN`); + for (const submission of submissions) { + const embedding = `[${submission.vector.map(Number).join(",")}]`; // stringify the floatvecotr + const sanitizedMetadata = this.sanitizeForJsonb(submission.metadata); + await connection.query( + `INSERT INTO "${Vastbase.tableName()}" (id, namespace, embedding, metadata) VALUES ($1, $2, $3, $4)`, + [submission.id, namespace, embedding, sanitizedMetadata] + ); + } + this.log(`Committing ${submissions.length} vectors to ${namespace}`); + await connection.query(`COMMIT`); + } catch (err) { + this.log( + `Rolling back ${submissions.length} vectors to ${namespace}`, + err + ); + await connection.query(`ROLLBACK`); + } + return true; + }, + + /** + * create a table if it doesn't exist + * @param {vsql.Client} connection + * @param {number} dimensions + * @returns + */ + createTableIfNotExists: async function (connection, dimensions = 384) { + this.log(`Creating embedding table with ${dimensions} dimensions`); + await connection.query(this.createTableSql(dimensions)); + return true; + }, + + /** + * Get the namespace from the database + * @param {vsql.Client} connection + * @param {string} namespace + * @returns {Promise<{name: string, vectorCount: number}>} + */ + namespace: async function (connection, namespace = null) { + if (!namespace) throw new Error("No namespace provided"); + const result = await connection.query( + `SELECT COUNT(id) FROM "${Vastbase.tableName()}" WHERE namespace = $1`, + [namespace] + ); + return { name: namespace, vectorCount: result.rows[0].count }; + }, + + /** + * Check if the namespace exists in the database + * @param {string} namespace + * @returns {Promise} + */ + hasNamespace: async function (namespace = null) { + if (!namespace) throw new Error("No namespace provided"); + let connection = null; + try { + connection = await this.connect(); + return await this.namespaceExists(connection, namespace); + } catch (err) { + return false; + } finally { + if (connection) await connection.end(); + } + }, + + /** + * Check if the namespace exists in the database + * @param {vsql.Client} connection + * @param {string} namespace + * @returns {Promise} + */ + namespaceExists: async function (connection, namespace = null) { + if (!namespace) throw new Error("No namespace provided"); + const result = await connection.query( + `SELECT COUNT(id) FROM "${Vastbase.tableName()}" WHERE namespace = $1 LIMIT 1`, + [namespace] + ); + return result.rows[0].count > 0; + }, + + /** + * Delete all vectors in the namespace + * @param {vsql.Client} connection + * @param {string} namespace + * @returns {Promise} + */ + deleteVectorsInNamespace: async function (connection, namespace = null) { + if (!namespace) throw new Error("No namespace provided"); + await connection.query( + `DELETE FROM "${Vastbase.tableName()}" WHERE namespace = $1`, + [namespace] + ); + return true; + }, + + addDocumentToNamespace: async function ( + namespace, + documentData = {}, + fullFilePath = null, + skipCache = false + ) { + const { DocumentVectors } = require("../../../models/vectors"); + const { + storeVectorResult, + cachedVectorInformation, + } = require("../../files"); + let connection = null; + + try { + const { pageContent, docId, ...metadata } = documentData; + if (!pageContent || pageContent.length == 0) return false; + connection = await this.connect(); + + this.log("Adding new vectorized document into namespace", namespace); + if (!skipCache) { + const cacheResult = await cachedVectorInformation(fullFilePath); + let vectorDimensions; + if (cacheResult.exists) { + const { chunks } = cacheResult; + const documentVectors = []; + const submissions = []; + + for (const chunk of chunks.flat()) { + if (!vectorDimensions) vectorDimensions = chunk.values.length; + const id = uuidv4(); + const { id: _id, ...metadata } = chunk.metadata; + documentVectors.push({ docId, vectorId: id }); + submissions.push({ id: id, vector: chunk.values, metadata }); + } + + await this.updateOrCreateCollection({ + connection, + submissions, + namespace, + dimensions: vectorDimensions, + }); + await DocumentVectors.bulkInsert(documentVectors); + return { vectorized: true, error: null }; + } + } + + // If we are here then we are going to embed and store a novel document. + // We have to do this manually as opposed to using LangChains `xyz.fromDocuments` + // because we then cannot atomically control our namespace to granularly find/remove documents + // from vectordb. + const { SystemSettings } = require("../../../models/systemSettings"); + const EmbedderEngine = getEmbeddingEngineSelection(); + const textSplitter = new TextSplitter({ + chunkSize: TextSplitter.determineMaxChunkSize( + await SystemSettings.getValueOrFallback({ + label: "text_splitter_chunk_size", + }), + EmbedderEngine?.embeddingMaxChunkLength + ), + chunkOverlap: await SystemSettings.getValueOrFallback( + { label: "text_splitter_chunk_overlap" }, + 20 + ), + chunkHeaderMeta: TextSplitter.buildHeaderMeta(metadata), + chunkPrefix: EmbedderEngine?.embeddingPrefix, + }); + const textChunks = await textSplitter.splitText(pageContent); + + this.log("Snippets created from document:", textChunks.length); + const documentVectors = []; + const vectors = []; + const submissions = []; + const vectorValues = await EmbedderEngine.embedChunks(textChunks); + let vectorDimensions; + + if (!!vectorValues && vectorValues.length > 0) { + for (const [i, vector] of vectorValues.entries()) { + if (!vectorDimensions) vectorDimensions = vector.length; + const vectorRecord = { + id: uuidv4(), + values: vector, + metadata: { ...metadata, text: textChunks[i] }, + }; + + vectors.push(vectorRecord); + submissions.push({ + id: vectorRecord.id, + vector: vectorRecord.values, + metadata: vectorRecord.metadata, + }); + documentVectors.push({ docId, vectorId: vectorRecord.id }); + } + } else { + throw new Error( + "Could not embed document chunks! This document will not be recorded." + ); + } + + if (vectors.length > 0) { + const chunks = []; + for (const chunk of toChunks(vectors, 500)) chunks.push(chunk); + + this.log("Inserting vectorized chunks into Vastbase collection."); + await this.updateOrCreateCollection({ + connection, + submissions, + namespace, + dimensions: vectorDimensions, + }); + await storeVectorResult(chunks, fullFilePath); + } + + await DocumentVectors.bulkInsert(documentVectors); + return { vectorized: true, error: null }; + } catch (err) { + this.log("addDocumentToNamespace", err.message); + return { vectorized: false, error: err.message }; + } finally { + if (connection) await connection.end(); + } + }, + + /** + * Delete a document from the namespace + * @param {string} namespace + * @param {string} docId + * @returns {Promise} + */ + deleteDocumentFromNamespace: async function (namespace, docId) { + if (!namespace) throw new Error("No namespace provided"); + if (!docId) throw new Error("No docId provided"); + + let connection = null; + try { + connection = await this.connect(); + const exists = await this.namespaceExists(connection, namespace); + if (!exists) + throw new Error( + `Vastbase:deleteDocumentFromNamespace - namespace ${namespace} does not exist.` + ); + + const { DocumentVectors } = require("../../../models/vectors"); + const vectorIds = (await DocumentVectors.where({ docId })).map( + (record) => record.vectorId + ); + if (vectorIds.length === 0) return; + + try { + await connection.query(`BEGIN`); + for (const vectorId of vectorIds) + await connection.query( + `DELETE FROM "${Vastbase.tableName()}" WHERE id = $1`, + [vectorId] + ); + await connection.query(`COMMIT`); + } catch (err) { + await connection.query(`ROLLBACK`); + throw err; + } + + this.log( + `Deleted ${vectorIds.length} vectors from namespace ${namespace}` + ); + return true; + } catch (err) { + this.log( + `Error deleting document from namespace ${namespace}: ${err.message}` + ); + return false; + } finally { + if (connection) await connection.end(); + } + }, + + performSimilaritySearch: async function ({ + namespace = null, + input = "", + LLMConnector = null, + similarityThreshold = 0.25, + topN = 4, + filterIdentifiers = [], + }) { + let connection = null; + if (!namespace || !input || !LLMConnector) + throw new Error("Invalid request to performSimilaritySearch."); + + try { + connection = await this.connect(); + const exists = await this.namespaceExists(connection, namespace); + if (!exists) { + this.log( + `The namespace ${namespace} does not exist or has no vectors. Returning empty results.` + ); + return { + contextTexts: [], + sources: [], + message: null, + }; + } + + const queryVector = await LLMConnector.embedTextInput(input); + const result = await this.similarityResponse({ + client: connection, + namespace, + queryVector, + similarityThreshold, + topN, + filterIdentifiers, + }); + + const { contextTexts, sourceDocuments } = result; + const sources = sourceDocuments.map((metadata, i) => { + return { metadata: { ...metadata, text: contextTexts[i] } }; + }); + return { + contextTexts, + sources: this.curateSources(sources), + message: false, + }; + } catch (err) { + return { error: err.message, success: false }; + } finally { + if (connection) await connection.end(); + } + }, + + "namespace-stats": async function (reqBody = {}) { + const { namespace = null } = reqBody; + if (!namespace) throw new Error("namespace required"); + if (!(await this.dbTableExists())) + return { message: "No table found in database" }; + + let connection = null; + try { + connection = await this.connect(); + if (!(await this.namespaceExists(connection, namespace))) + throw new Error("Namespace by that name does not exist."); + const stats = await this.namespace(connection, namespace); + return stats + ? stats + : { message: "No stats were able to be fetched from DB for namespace" }; + } catch (err) { + return { + message: `Error fetching stats for namespace ${namespace}: ${err.message}`, + }; + } finally { + if (connection) await connection.end(); + } + }, + + "delete-namespace": async function (reqBody = {}) { + const { namespace = null } = reqBody; + if (!namespace) throw new Error("No namespace provided"); + + let connection = null; + try { + const existingCount = await this.namespaceCount(namespace); + if (existingCount === 0) + return { + message: `Namespace ${namespace} does not exist or has no vectors.`, + }; + + connection = await this.connect(); + await this.deleteVectorsInNamespace(connection, namespace); + return { + message: `Namespace ${namespace} was deleted along with ${existingCount} vectors.`, + }; + } catch (err) { + return { + message: `Error deleting namespace ${namespace}: ${err.message}`, + }; + } finally { + if (connection) await connection.end(); + } + }, + + /** + * Reset the entire vector database table associated with anythingllm + * @returns {Promise<{reset: boolean}>} + */ + reset: async function () { + let connection = null; + try { + connection = await this.connect(); + await connection.query(`DROP TABLE IF EXISTS "${Vastbase.tableName()}"`); + return { reset: true }; + } catch (err) { + return { reset: false }; + } finally { + if (connection) await connection.end(); + } + }, + + curateSources: function (sources = []) { + const documents = []; + for (const source of sources) { + const { text, vector: _v, _distance: _d, ...rest } = source; + const metadata = rest.hasOwnProperty("metadata") ? rest.metadata : rest; + if (Object.keys(metadata).length > 0) { + documents.push({ + ...metadata, + ...(text ? { text } : {}), + }); + } + } + + return documents; + }, +}; + +module.exports.Vastbase = Vastbase; diff --git a/server/utils/vectorStore/resetAllVectorStores.js b/server/utils/vectorStore/resetAllVectorStores.js index f47fc970b2c..09f573ec447 100644 --- a/server/utils/vectorStore/resetAllVectorStores.js +++ b/server/utils/vectorStore/resetAllVectorStores.js @@ -39,6 +39,14 @@ async function resetAllVectorStores({ vectorDbKey }) { the dimension value of an existing vector column. */ await VectorDb.reset(); + } else if (vectorDbKey === "vastbase"){ + /* + Vastbase has a reset method that drops the entire embedding table + which is required since if this function is called we will need to + reset the embedding column VECTOR dimension value and you cannot change + the dimension value of an existing vector column. + */ + await VectorDb.reset(); } else { for (const workspace of workspaces) { try {