From b39aa048d1213df49b90a88dfeead9c24e7a1d1c Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Wed, 13 Aug 2025 13:01:18 +0200 Subject: [PATCH 1/7] feat: add BackupGetter to the public API --- src/backup/index.ts | 4 ++ src/backup/journey.test.ts | 105 ++++++++++++++++++++----------------- 2 files changed, 60 insertions(+), 49 deletions(-) diff --git a/src/backup/index.ts b/src/backup/index.ts index 9b395cd6..f7c882fe 100644 --- a/src/backup/index.ts +++ b/src/backup/index.ts @@ -1,6 +1,7 @@ import Connection from '../connection/index.js'; import BackupCreateStatusGetter from './backupCreateStatusGetter.js'; import BackupCreator from './backupCreator.js'; +import BackupGetter from './backupGetter.js'; import BackupRestoreStatusGetter from './backupRestoreStatusGetter.js'; import BackupRestorer from './backupRestorer.js'; @@ -13,6 +14,7 @@ export interface Backup { createStatusGetter: () => BackupCreateStatusGetter; restorer: () => BackupRestorer; restoreStatusGetter: () => BackupRestoreStatusGetter; + getter: () => BackupGetter; } const backup = (client: Connection): Backup => { @@ -21,11 +23,13 @@ const backup = (client: Connection): Backup => { createStatusGetter: () => new BackupCreateStatusGetter(client), restorer: () => new BackupRestorer(client, new BackupRestoreStatusGetter(client)), restoreStatusGetter: () => new BackupRestoreStatusGetter(client), + getter: () => new BackupGetter(client), }; }; export default backup; export { default as BackupCreateStatusGetter } from './backupCreateStatusGetter.js'; export { default as BackupCreator } from './backupCreator.js'; +export { default as BackupGetter } from './backupGetter.js'; export { default as BackupRestoreStatusGetter } from './backupRestoreStatusGetter.js'; export { default as BackupRestorer } from './backupRestorer.js'; diff --git a/src/backup/journey.test.ts b/src/backup/journey.test.ts index fc04ff09..554110c1 100644 --- a/src/backup/journey.test.ts +++ b/src/backup/journey.test.ts @@ -7,6 +7,7 @@ import { import weaviate, { WeaviateClient } from '../v2/index.js'; import { Backend } from './index.js'; +import { requireAtLeast } from '../../test/version'; const { createTestFoodSchemaAndData, cleanupTestFood, @@ -950,55 +951,61 @@ describe('fails restoring backup with invalid compression config', () => { it('cleans up', () => cleanupTestFood(client)); }); -// describe("get all exising backups", () => { -// const BACKEND: Backend = 'filesystem'; -// const BACKUP_ID = randomBackupId() -// const BACKUP_ID_PIZZA = BACKUP_ID + "-pizza"; -// const BACKUP_ID_SOUP = BACKUP_ID + "-soup"; - -// const client = weaviate.client({ -// scheme: "http", -// host: "localhost:8080", -// }); - -// it("sets up", () => createTestFoodSchemaAndData(client)); - -// it("creates backup pizza", () => { -// return client.backup.creator() -// .withIncludeClassNames(PIZZA_CLASS_NAME) -// .withBackend(BACKEND) -// .withBackupId(BACKUP_ID_PIZZA) -// .withWaitForCompletion(true) -// .do() -// .catch((err: any) => {throw new Error("should not fail on create backup: " + err)}); -// }); - -// it("creates backup soup", () => { -// return client.backup.creator() -// .withIncludeClassNames(SOUP_CLASS_NAME) -// .withBackend(BACKEND) -// .withBackupId(BACKUP_ID_SOUP) -// .withWaitForCompletion(true) -// .do() -// .catch((err: any) => {throw new Error("should not fail on create backup: " + err)}); -// }); - -// it("get all", () => { -// return client.backup.getter() -// .withBackend(BACKEND) -// .do() -// .then(allResponse => { -// expect(allResponse).toHaveLength(2); -// expect(allResponse).toEqual(expect.arrayContaining([ -// expect.objectContaining({id: BACKUP_ID_PIZZA}), -// expect.objectContaining({id: BACKUP_ID_SOUP}), -// ])); -// }) -// .catch((err: any) => {throw new Error("should not fail on getting all: " + err)}); -// }); - -// it("cleans up", () => cleanupTestFood(client)); -// }); +requireAtLeast(1, 30, 0).describe('get all exising backups', () => { + const BACKEND: Backend = 'filesystem'; + const BACKUP_ID = randomBackupId(); + const BACKUP_ID_PIZZA = BACKUP_ID + '-pizza'; + const BACKUP_ID_SOUP = BACKUP_ID + '-soup'; + + const client = weaviate.client({ + scheme: 'http', + host: 'localhost:8080', + }); + + it('sets up', () => createTestFoodSchemaAndData(client)); + + it('creates backup pizza', () => { + return client.backup + .creator() + .withIncludeClassNames(PIZZA_CLASS_NAME) + .withBackend(BACKEND) + .withBackupId(BACKUP_ID_PIZZA) + .withWaitForCompletion(true) + .do() + .catch((err: any) => { + throw new Error('should not fail on create backup: ' + err); + }); + }); + + it('creates backup soup', () => { + return client.backup + .creator() + .withIncludeClassNames(SOUP_CLASS_NAME) + .withBackend(BACKEND) + .withBackupId(BACKUP_ID_SOUP) + .withWaitForCompletion(true) + .do() + .catch((err: any) => { + throw new Error('should not fail on create backup: ' + err); + }); + }); + + it('get all', () => { + return client.backup + .getter() + .withBackend(BACKEND) + .do() + .then((allResponse) => { + const relevant = allResponse.filter((b) => b.id === BACKUP_ID_SOUP || b.id === BACKUP_ID_PIZZA); + expect(relevant).toHaveLength(2); + }) + .catch((err: any) => { + throw new Error('should not fail on getting all: ' + err); + }); + }); + + it('cleans up', () => cleanupTestFood(client)); +}); function assertThatAllPizzasExist(client: WeaviateClient) { return assertThatAllFoodObjectsExist(client, 'Pizza', 4); From a0fde37798a1578360d2aa17f5b91e5ac1243408 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Thu, 14 Aug 2025 10:08:27 +0200 Subject: [PATCH 2/7] test: test list backups for v1.32+ --- src/backup/journey.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backup/journey.test.ts b/src/backup/journey.test.ts index 554110c1..e438b4a1 100644 --- a/src/backup/journey.test.ts +++ b/src/backup/journey.test.ts @@ -951,7 +951,7 @@ describe('fails restoring backup with invalid compression config', () => { it('cleans up', () => cleanupTestFood(client)); }); -requireAtLeast(1, 30, 0).describe('get all exising backups', () => { +requireAtLeast(1, 32, 0).describe('get all exising backups', () => { const BACKEND: Backend = 'filesystem'; const BACKUP_ID = randomBackupId(); const BACKUP_ID_PIZZA = BACKUP_ID + '-pizza'; From 600be3423cb081ba8c36c3c469f9f6798d39757b Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Thu, 21 Aug 2025 21:41:07 +0200 Subject: [PATCH 3/7] feat: add list backups to v3 API --- src/backup/journey.test.ts | 57 ---------------------- src/collections/backup/client.ts | 15 +++++- src/collections/backup/integration.test.ts | 56 +++++++++++++++++++-- 3 files changed, 66 insertions(+), 62 deletions(-) diff --git a/src/backup/journey.test.ts b/src/backup/journey.test.ts index e438b4a1..6d7d9f8c 100644 --- a/src/backup/journey.test.ts +++ b/src/backup/journey.test.ts @@ -7,7 +7,6 @@ import { import weaviate, { WeaviateClient } from '../v2/index.js'; import { Backend } from './index.js'; -import { requireAtLeast } from '../../test/version'; const { createTestFoodSchemaAndData, cleanupTestFood, @@ -951,62 +950,6 @@ describe('fails restoring backup with invalid compression config', () => { it('cleans up', () => cleanupTestFood(client)); }); -requireAtLeast(1, 32, 0).describe('get all exising backups', () => { - const BACKEND: Backend = 'filesystem'; - const BACKUP_ID = randomBackupId(); - const BACKUP_ID_PIZZA = BACKUP_ID + '-pizza'; - const BACKUP_ID_SOUP = BACKUP_ID + '-soup'; - - const client = weaviate.client({ - scheme: 'http', - host: 'localhost:8080', - }); - - it('sets up', () => createTestFoodSchemaAndData(client)); - - it('creates backup pizza', () => { - return client.backup - .creator() - .withIncludeClassNames(PIZZA_CLASS_NAME) - .withBackend(BACKEND) - .withBackupId(BACKUP_ID_PIZZA) - .withWaitForCompletion(true) - .do() - .catch((err: any) => { - throw new Error('should not fail on create backup: ' + err); - }); - }); - - it('creates backup soup', () => { - return client.backup - .creator() - .withIncludeClassNames(SOUP_CLASS_NAME) - .withBackend(BACKEND) - .withBackupId(BACKUP_ID_SOUP) - .withWaitForCompletion(true) - .do() - .catch((err: any) => { - throw new Error('should not fail on create backup: ' + err); - }); - }); - - it('get all', () => { - return client.backup - .getter() - .withBackend(BACKEND) - .do() - .then((allResponse) => { - const relevant = allResponse.filter((b) => b.id === BACKUP_ID_SOUP || b.id === BACKUP_ID_PIZZA); - expect(relevant).toHaveLength(2); - }) - .catch((err: any) => { - throw new Error('should not fail on getting all: ' + err); - }); - }); - - it('cleans up', () => cleanupTestFood(client)); -}); - function assertThatAllPizzasExist(client: WeaviateClient) { return assertThatAllFoodObjectsExist(client, 'Pizza', 4); } diff --git a/src/collections/backup/client.ts b/src/collections/backup/client.ts index b24f12bd..7c22fc6c 100644 --- a/src/collections/backup/client.ts +++ b/src/collections/backup/client.ts @@ -30,7 +30,7 @@ import { BackupStatusReturn, } from './types.js'; -export const backup = (connection: Connection) => { +export const backup = (connection: Connection): Backup => { const parseStatus = (res: BackupCreateStatusResponse | BackupRestoreResponse): BackupStatusReturn => { if (res.id === undefined) { throw new WeaviateUnexpectedResponseError('Backup ID is undefined in response'); @@ -202,6 +202,9 @@ export const backup = (connection: Connection) => { } : parseResponse(res); }, + list: (backend: Backend): Promise => { + return connection.get(`/backups/${backend}`); + }, }; }; @@ -251,4 +254,14 @@ export interface Backup { * @throws {WeaviateBackupCanceled} If the backup restoration is canceled. */ restore(args: BackupArgs): Promise; + + /** List existing backups (completed and in-progress) created in a given backend. + * + * @param {Backend} backend Backend whence to list backups. + * @returns {Promise} The response from Weaviate. + * @throws {WeaviateInvalidInputError} If the input is invalid. + * @throws {WeaviateBackupFailed} If the backup restoration fails. + * @throws {WeaviateBackupCanceled} If the backup restoration is canceled. + * */ + list(backend: Backend): Promise; } diff --git a/src/collections/backup/integration.test.ts b/src/collections/backup/integration.test.ts index 66b38b2d..4403679c 100644 --- a/src/collections/backup/integration.test.ts +++ b/src/collections/backup/integration.test.ts @@ -1,6 +1,7 @@ /* eslint-disable @typescript-eslint/no-non-null-assertion */ /* eslint-disable @typescript-eslint/no-non-null-asserted-optional-chain */ /* eslint-disable no-await-in-loop */ +import { requireAtLeast } from '../../../test/version.js'; import { Backend } from '../../backup/index.js'; import weaviate, { Collection, WeaviateClient } from '../../index.js'; @@ -88,6 +89,23 @@ describe('Integration testing of backups', () => { backend: res.backend as 'filesystem', }); expect(status).not.toBe('SUCCESS'); // can be 'STARTED' or 'TRANSFERRING' depending on the speed of the test machine + + // wait to complete so that other tests can run without colliding with Weaviate's lack of simultaneous backups + let wait = true; + while (wait) { + const { status, error } = await collection.backup.getCreateStatus({ + backupId: res.id as string, + backend: res.backend as Backend, + }); + if (status === 'SUCCESS') { + wait = false; + } + if (status === 'FAILED') { + throw new Error(`Backup creation failed: ${error}`); + } + await new Promise((resolve) => setTimeout(resolve, 1000)); + } + return collection; }; @@ -98,8 +116,38 @@ describe('Integration testing of backups', () => { .then(getCollection) .then(testCollectionWaitForCompletion) .then(testCollectionNoWaitForCompletion)); -}); -function randomBackupId() { - return 'backup-id-' + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); -} + requireAtLeast(1, 32, 0).it('get all exising backups', async () => { + await clientPromise.then(async (client) => { + await client.collections.create({ name: 'TestListBackups' }).then((col) => col.data.insert()); + + const wantBackups: string[] = []; + for (let i = 0; i < 3; i++) { + wantBackups.push( + await client.backup + .create({ + backupId: randomBackupId(), + backend: 'filesystem', + includeCollections: ['TestListBackups'], + waitForCompletion: true, + }) + .then((res) => { + return res.id; + }) + ); + } + + const gotBackups: string[] = await client.backup + .list('filesystem') + .then((res) => res.map((bu) => bu.id)); + + // There may be other backups created in other tests; + expect(gotBackups.length).toBeGreaterThanOrEqual(wantBackups.length); + expect(gotBackups).toEqual(expect.arrayContaining(wantBackups)); + }); + }); + + function randomBackupId() { + return 'backup-id-' + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); + } +}); From c572e3cc79fd120cd1b4ac7e5fe7035828c8ef52 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Thu, 21 Aug 2025 21:44:04 +0200 Subject: [PATCH 4/7] chore: remove BackupGetter from legacy client version --- src/backup/backupGetter.ts | 36 ------------------------------------ src/backup/index.ts | 4 ---- 2 files changed, 40 deletions(-) delete mode 100644 src/backup/backupGetter.ts diff --git a/src/backup/backupGetter.ts b/src/backup/backupGetter.ts deleted file mode 100644 index 71c1142f..00000000 --- a/src/backup/backupGetter.ts +++ /dev/null @@ -1,36 +0,0 @@ -import Connection from '../connection/index.js'; -import { WeaviateInvalidInputError } from '../errors.js'; -import { BackupCreateResponse } from '../openapi/types.js'; -import { CommandBase } from '../validation/commandBase.js'; -import { Backend } from './index.js'; -import { validateBackend } from './validation.js'; - -export default class BackupGetter extends CommandBase { - private backend?: Backend; - - constructor(client: Connection) { - super(client); - } - - withBackend(backend: Backend) { - this.backend = backend; - return this; - } - - validate = (): void => { - this.addErrors(validateBackend(this.backend)); - }; - - do = (): Promise => { - this.validate(); - if (this.errors.length > 0) { - return Promise.reject(new WeaviateInvalidInputError('invalid usage: ' + this.errors.join(', '))); - } - - return this.client.get(this._path()); - }; - - private _path = (): string => { - return `/backups/${this.backend}`; - }; -} diff --git a/src/backup/index.ts b/src/backup/index.ts index f7c882fe..9b395cd6 100644 --- a/src/backup/index.ts +++ b/src/backup/index.ts @@ -1,7 +1,6 @@ import Connection from '../connection/index.js'; import BackupCreateStatusGetter from './backupCreateStatusGetter.js'; import BackupCreator from './backupCreator.js'; -import BackupGetter from './backupGetter.js'; import BackupRestoreStatusGetter from './backupRestoreStatusGetter.js'; import BackupRestorer from './backupRestorer.js'; @@ -14,7 +13,6 @@ export interface Backup { createStatusGetter: () => BackupCreateStatusGetter; restorer: () => BackupRestorer; restoreStatusGetter: () => BackupRestoreStatusGetter; - getter: () => BackupGetter; } const backup = (client: Connection): Backup => { @@ -23,13 +21,11 @@ const backup = (client: Connection): Backup => { createStatusGetter: () => new BackupCreateStatusGetter(client), restorer: () => new BackupRestorer(client, new BackupRestoreStatusGetter(client)), restoreStatusGetter: () => new BackupRestoreStatusGetter(client), - getter: () => new BackupGetter(client), }; }; export default backup; export { default as BackupCreateStatusGetter } from './backupCreateStatusGetter.js'; export { default as BackupCreator } from './backupCreator.js'; -export { default as BackupGetter } from './backupGetter.js'; export { default as BackupRestoreStatusGetter } from './backupRestoreStatusGetter.js'; export { default as BackupRestorer } from './backupRestorer.js'; From 02a1d8c629b25fdbd7581aa24d83ad4f83a066f2 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Thu, 21 Aug 2025 21:45:43 +0200 Subject: [PATCH 5/7] chore: fix docstring: --- src/collections/backup/client.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/collections/backup/client.ts b/src/collections/backup/client.ts index 7c22fc6c..8674c796 100644 --- a/src/collections/backup/client.ts +++ b/src/collections/backup/client.ts @@ -197,9 +197,9 @@ export const backup = (connection: Connection): Backup => { } return status ? { - ...parseResponse(res), - ...status, - } + ...parseResponse(res), + ...status, + } : parseResponse(res); }, list: (backend: Backend): Promise => { @@ -259,9 +259,6 @@ export interface Backup { * * @param {Backend} backend Backend whence to list backups. * @returns {Promise} The response from Weaviate. - * @throws {WeaviateInvalidInputError} If the input is invalid. - * @throws {WeaviateBackupFailed} If the backup restoration fails. - * @throws {WeaviateBackupCanceled} If the backup restoration is canceled. * */ list(backend: Backend): Promise; } From 0b996b18048a80ebd5642102ad7221d3e92129f5 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Thu, 21 Aug 2025 21:48:30 +0200 Subject: [PATCH 6/7] chore: lint and format --- src/collections/backup/client.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/collections/backup/client.ts b/src/collections/backup/client.ts index 8674c796..6557f070 100644 --- a/src/collections/backup/client.ts +++ b/src/collections/backup/client.ts @@ -197,9 +197,9 @@ export const backup = (connection: Connection): Backup => { } return status ? { - ...parseResponse(res), - ...status, - } + ...parseResponse(res), + ...status, + } : parseResponse(res); }, list: (backend: Backend): Promise => { From 8b2b490142e46d1b5c038e873e44ba043e9676b7 Mon Sep 17 00:00:00 2001 From: dyma solovei Date: Thu, 21 Aug 2025 21:50:36 +0200 Subject: [PATCH 7/7] test: simplify callback --- src/collections/backup/integration.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/collections/backup/integration.test.ts b/src/collections/backup/integration.test.ts index 4403679c..4cbd4554 100644 --- a/src/collections/backup/integration.test.ts +++ b/src/collections/backup/integration.test.ts @@ -131,9 +131,7 @@ describe('Integration testing of backups', () => { includeCollections: ['TestListBackups'], waitForCompletion: true, }) - .then((res) => { - return res.id; - }) + .then((res) => res.id) ); }