From 6bea27d05ba32c870f8d3252872be01c861bfb7d Mon Sep 17 00:00:00 2001 From: SevenWaysDP Date: Fri, 7 Feb 2025 13:26:23 +0100 Subject: [PATCH 1/2] feat(database): add migration creation functionality and related tests --- .../server/src/config/mikro-orm-cli.config.ts | 2 +- .../database-management.service.spec.ts | 14 ++++++- .../management/database-management.service.ts | 34 ++++++++++------- .../database-management.console.spec.ts | 11 +++++- .../console/database-management.console.ts | 20 +++++++--- .../uc/database-management.uc.spec.ts | 5 +++ .../management/uc/database-management.uc.ts | 38 +++++++++++-------- package.json | 10 +---- 8 files changed, 88 insertions(+), 46 deletions(-) diff --git a/apps/server/src/config/mikro-orm-cli.config.ts b/apps/server/src/config/mikro-orm-cli.config.ts index 72a5b34502b..d1bb45a05f5 100644 --- a/apps/server/src/config/mikro-orm-cli.config.ts +++ b/apps/server/src/config/mikro-orm-cli.config.ts @@ -22,7 +22,7 @@ export const mikroOrmCliConfig: MikroOrmModuleSyncOptions = { migrations: { tableName: 'migrations', // name of database table with log of executed transactions path: migrationsPath, // path to the folder with migrations - pathTs: migrationsPath, // path to the folder with TS migrations (if used, we should put path to compiled files in `path`) + pathTs: migrationsPath.replace('/dist/apps/server/migrations/mikro-orm', '/apps/server/src/migrations/mikro-orm'), // path to the folder with TS migrations (if used, we should put path to compiled files in `path`) glob: '!(*.d).{js,ts}', // how to match migration files (all .js and .ts files, but not .d.ts) transactional: false, // wrap each migration in a transaction disableForeignKeys: true, // wrap statements with `set foreign_key_checks = 0` or equivalent diff --git a/apps/server/src/infra/database/management/database-management.service.spec.ts b/apps/server/src/infra/database/management/database-management.service.spec.ts index 99ce23b0836..ebcb1fc2289 100644 --- a/apps/server/src/infra/database/management/database-management.service.spec.ts +++ b/apps/server/src/infra/database/management/database-management.service.spec.ts @@ -1,4 +1,5 @@ -import { MikroORM } from '@mikro-orm/core'; +import { createMock, DeepMocked } from '@golevelup/ts-jest'; +import { IMigrator, MikroORM } from '@mikro-orm/core'; import { EntityManager, ObjectId } from '@mikro-orm/mongodb'; import { Test, TestingModule } from '@nestjs/testing'; import { createCollections } from '@testing/create-collections'; @@ -168,4 +169,15 @@ describe('DatabaseManagementService', () => { expect(spy).toHaveBeenCalled(); }); }); + + describe('When call migrationCreate()', () => { + it('should call migrator.createInitialMigration()', async () => { + const spyGetMigrator = jest.spyOn(orm, 'getMigrator'); + const spyMigrator: DeepMocked = createMock(); + spyGetMigrator.mockReturnValue(spyMigrator); + + await service.migrationCreate(); + expect(spyMigrator.createInitialMigration).toHaveBeenCalled(); + }); + }); }); diff --git a/apps/server/src/infra/database/management/database-management.service.ts b/apps/server/src/infra/database/management/database-management.service.ts index 83063666aa9..e40182da19b 100644 --- a/apps/server/src/infra/database/management/database-management.service.ts +++ b/apps/server/src/infra/database/management/database-management.service.ts @@ -14,12 +14,12 @@ export class DatabaseManagementService { return connection; } - getDatabaseCollection(collectionName: string): Collection { + public getDatabaseCollection(collectionName: string): Collection { const collection = this.db.collection(collectionName); return collection; } - async importCollection(collectionName: string, jsonDocuments: unknown[]): Promise { + public async importCollection(collectionName: string, jsonDocuments: unknown[]): Promise { if (jsonDocuments.length === 0) { return 0; } @@ -31,19 +31,19 @@ export class DatabaseManagementService { return insertedCount; } - async findDocumentsOfCollection(collectionName: string): Promise { + public async findDocumentsOfCollection(collectionName: string): Promise { const collection = this.getDatabaseCollection(collectionName); const documents = (await collection.find({}).toArray()) as unknown[]; return documents; } - async clearCollection(collectionName: string): Promise { + public async clearCollection(collectionName: string): Promise { const collection = this.getDatabaseCollection(collectionName); const { deletedCount } = await collection.deleteMany({}); return deletedCount || 0; } - async getCollectionNames(): Promise { + public async getCollectionNames(): Promise { const collections = (await this.db.listCollections(undefined, { nameOnly: true }).toArray()) as unknown[] as { name: string; }[]; @@ -51,44 +51,50 @@ export class DatabaseManagementService { return collectionNames; } - async collectionExists(collectionName: string): Promise { + public async collectionExists(collectionName: string): Promise { const collections = await this.getCollectionNames(); const result = collections.includes(collectionName); return result; } - async createCollection(collectionName: string): Promise { + public async createCollection(collectionName: string): Promise { await this.db.createCollection(collectionName); } - async dropCollection(collectionName: string): Promise { + public async dropCollection(collectionName: string): Promise { await this.db.dropCollection(collectionName); } - async syncIndexes(): Promise { - return this.orm.getSchemaGenerator().ensureIndexes(); + public async syncIndexes(): Promise { + await this.orm.getSchemaGenerator().ensureIndexes(); } - async migrationUp(from?: string, to?: string, only?: string): Promise { + public async migrationUp(from?: string, to?: string, only?: string): Promise { const migrator = this.orm.getMigrator(); const params = this.migrationParams(only, from, to); await migrator.up(params); } - async migrationDown(from?: string, to?: string, only?: string): Promise { + public async migrationDown(from?: string, to?: string, only?: string): Promise { const migrator = this.orm.getMigrator(); const params = this.migrationParams(only, from, to); await migrator.down(params); } - async migrationPending(): Promise { + public async migrationPending(): Promise { const migrator = this.orm.getMigrator(); const pendingMigrations = await migrator.getPendingMigrations(); return pendingMigrations; } - private migrationParams(only?: string, from?: string, to?: string) { + public async migrationCreate(): Promise { + const migrator = this.orm.getMigrator(); + + await migrator.createInitialMigration(); + } + + private migrationParams(only?: string, from?: string, to?: string): MigrateOptions { const params: MigrateOptions = {}; if (only) { params.migrations = [only]; diff --git a/apps/server/src/modules/management/console/database-management.console.spec.ts b/apps/server/src/modules/management/console/database-management.console.spec.ts index 5e08af4edd5..51bb344ec58 100644 --- a/apps/server/src/modules/management/console/database-management.console.spec.ts +++ b/apps/server/src/modules/management/console/database-management.console.spec.ts @@ -1,6 +1,6 @@ import { createMock, DeepMocked } from '@golevelup/ts-jest'; -import { Test, TestingModule } from '@nestjs/testing'; import { ConsoleWriterService } from '@infra/console'; +import { Test, TestingModule } from '@nestjs/testing'; import { DatabaseManagementUc } from '../uc/database-management.uc'; import { DatabaseManagementConsole } from './database-management.console'; @@ -100,16 +100,25 @@ describe('DatabaseManagementConsole', () => { expect(consoleInfoSpy).toHaveBeenCalledWith('migration up is completed'); expect(databaseManagementUc.migrationUp).toHaveBeenCalled(); }); + it('should migrate down', async () => { await service.migration({ down: true }); expect(consoleInfoSpy).toHaveBeenCalledWith('migration down is completed'); expect(databaseManagementUc.migrationDown).toHaveBeenCalled(); }); + it('should check pending migrations', async () => { await service.migration({ pending: true }); expect(consoleInfoSpy).toHaveBeenCalledWith(expect.stringContaining('Pending:')); expect(databaseManagementUc.migrationPending).toHaveBeenCalled(); }); + + it('should check create migrations', async () => { + await service.migration({ create: true }); + expect(consoleInfoSpy).toHaveBeenCalledWith('migration created'); + expect(databaseManagementUc.migrationCreate).toHaveBeenCalled(); + }); + it('should no migrate if no param specified', async () => { await service.migration({}); expect(consoleErrorSpy).toHaveBeenCalledWith('no migration option was given'); diff --git a/apps/server/src/modules/management/console/database-management.console.ts b/apps/server/src/modules/management/console/database-management.console.ts index 1b14fb37bd5..2f2e12d3d62 100644 --- a/apps/server/src/modules/management/console/database-management.console.ts +++ b/apps/server/src/modules/management/console/database-management.console.ts @@ -15,6 +15,7 @@ interface MigrationOptions { to?: string; only?: string; pending?: boolean; + create?: boolean; } @Console({ command: 'database', description: 'database setup console' }) @@ -37,7 +38,7 @@ export class DatabaseManagementConsole { ], description: 'reset database collections with seed data from filesystem', }) - async seedCollections(options: Options): Promise { + public async seedCollections(options: Options): Promise { const filter = options?.collection ? [options.collection] : undefined; const collections = options.onlyfactories @@ -64,7 +65,7 @@ export class DatabaseManagementConsole { ], description: 'export database collections to filesystem', }) - async exportCollections(options: Options): Promise { + public async exportCollections(options: Options): Promise { const filter = options?.collection ? [options.collection] : undefined; const toSeedFolder = options?.override ? true : undefined; const collections = await this.databaseManagementUc.exportCollectionsToFileSystem(filter, toSeedFolder); @@ -78,7 +79,7 @@ export class DatabaseManagementConsole { options: [], description: 'sync indexes from nest and mikroorm', }) - async syncIndexes(): Promise { + public async syncIndexes(): Promise { await this.databaseManagementUc.syncIndexes(); const report = 'sync of indexes is completed'; this.consoleWriter.info(report); @@ -118,12 +119,17 @@ export class DatabaseManagementConsole { required: false, description: 'list pending migrations', }, + { + flags: '--create', + required: false, + description: 'create a new migration', + }, ], description: 'Execute MikroOrm migration up/down', }) - async migration(migrationOptions: MigrationOptions): Promise { + public async migration(migrationOptions: MigrationOptions): Promise { let report = 'no migration option was given'; - if (!migrationOptions.up && !migrationOptions.down && !migrationOptions.pending) { + if (!migrationOptions.up && !migrationOptions.down && !migrationOptions.pending && !migrationOptions.create) { this.consoleWriter.error(report); return report; } @@ -139,6 +145,10 @@ export class DatabaseManagementConsole { const pendingMigrations = await this.databaseManagementUc.migrationPending(); report = `Pending: ${JSON.stringify(pendingMigrations.map((migration) => migration.name))}`; } + if (migrationOptions.create) { + await this.databaseManagementUc.migrationCreate(); + report = 'migration created'; + } this.consoleWriter.info(report); return report; diff --git a/apps/server/src/modules/management/uc/database-management.uc.spec.ts b/apps/server/src/modules/management/uc/database-management.uc.spec.ts index 363e7d3b042..d50ad8166a4 100644 --- a/apps/server/src/modules/management/uc/database-management.uc.spec.ts +++ b/apps/server/src/modules/management/uc/database-management.uc.spec.ts @@ -701,5 +701,10 @@ describe('DatabaseManagementService', () => { await uc.migrationPending(); expect(dbService.migrationPending).toHaveBeenCalled(); }); + it('should call migrationCreate', async () => { + jest.spyOn(dbService, 'migrationDown').mockImplementation(); + await uc.migrationCreate(); + expect(dbService.migrationCreate).toHaveBeenCalled(); + }); }); }); diff --git a/apps/server/src/modules/management/uc/database-management.uc.ts b/apps/server/src/modules/management/uc/database-management.uc.ts index 706b55d3d9b..79c8c96db67 100644 --- a/apps/server/src/modules/management/uc/database-management.uc.ts +++ b/apps/server/src/modules/management/uc/database-management.uc.ts @@ -57,7 +57,7 @@ export class DatabaseManagementUc { /** * setup dir with json files */ - private getSeedFolder() { + private getSeedFolder(): string { return this.fileSystemAdapter.joinPath(this.baseDir, 'setup'); } @@ -65,7 +65,7 @@ export class DatabaseManagementUc { * export folder name based on current date * @returns */ - private getTargetFolder(toSeedFolder?: boolean) { + private getTargetFolder(toSeedFolder?: boolean): string { if (toSeedFolder === true) { const targetFolder = this.getSeedFolder(); return targetFolder; @@ -118,7 +118,7 @@ export class DatabaseManagementUc { source: 'files' | 'database', folder: string, collectionNameFilter?: string[] - ) { + ): Promise { let allCollectionsWithFilePaths: CollectionFilePath[] = []; // load all available collections from source @@ -151,7 +151,7 @@ export class DatabaseManagementUc { return allCollectionsWithFilePaths; } - private async dropCollectionIfExists(collectionName: string) { + private async dropCollectionIfExists(collectionName: string): Promise { const collectionExists = await this.databaseManagementService.collectionExists(collectionName); if (collectionExists) { // clear existing documents, if collection exists @@ -162,7 +162,7 @@ export class DatabaseManagementUc { } } - async seedDatabaseCollectionsFromFactories(collections?: string[]): Promise { + public async seedDatabaseCollectionsFromFactories(collections?: string[]): Promise { const promises = generateSeedData((s: string) => this.injectEnvVars(s)) .filter((data) => { if (collections && collections.length > 0) { @@ -266,7 +266,7 @@ export class DatabaseManagementUc { * @param toSeedFolder optional override existing seed data files * @returns the list of collection names exported */ - async exportCollectionsToFileSystem(collections?: string[], toSeedFolder?: boolean): Promise { + public async exportCollectionsToFileSystem(collections?: string[], toSeedFolder?: boolean): Promise { const targetFolder = this.getTargetFolder(toSeedFolder); await this.fileSystemAdapter.createDir(targetFolder); // detect collections to export based on database collections @@ -301,7 +301,7 @@ export class DatabaseManagementUc { /** * Updates the indexes in the database based on definitions in entities */ - async syncIndexes(): Promise { + public async syncIndexes(): Promise { await this.createUserSearchIndex(); return this.databaseManagementService.syncIndexes(); } @@ -357,7 +357,7 @@ export class DatabaseManagementUc { return json; } - private resolvePlaceholder(placeholder: string) { + private resolvePlaceholder(placeholder: string): string { if (Configuration.has(placeholder)) { return Configuration.get(placeholder) as string; } @@ -369,13 +369,13 @@ export class DatabaseManagementUc { return ''; } - private encryptSecrets(collectionName: string, jsonDocuments: unknown[]) { + private encryptSecrets(collectionName: string, jsonDocuments: unknown[]): void { if (collectionName === systemsCollectionName) { this.encryptSecretsInSystems(jsonDocuments as SystemEntity[]); } } - private encryptSecretsInSystems(systems: SystemEntity[]) { + private encryptSecretsInSystems(systems: SystemEntity[]): SystemEntity[] { systems.forEach((system) => { if (system.oauthConfig) { system.oauthConfig.clientSecret = this.defaultEncryptionService.encrypt(system.oauthConfig.clientSecret); @@ -397,7 +397,7 @@ export class DatabaseManagementUc { * Manual replacement with the intend placeholders or value is mandatory. * Currently, this affects system and storageproviders collections. */ - private removeSecrets(collectionName: string, jsonDocuments: unknown[]) { + private removeSecrets(collectionName: string, jsonDocuments: unknown[]): void { if (collectionName === systemsCollectionName) { this.removeSecretsFromSystems(jsonDocuments as SystemEntity[]); } @@ -406,14 +406,14 @@ export class DatabaseManagementUc { } } - private removeSecretsFromStorageproviders(storageProviders: StorageProviderEntity[]) { + private removeSecretsFromStorageproviders(storageProviders: StorageProviderEntity[]): void { storageProviders.forEach((storageProvider) => { storageProvider.accessKeyId = defaultSecretReplacementHintText; storageProvider.secretAccessKey = defaultSecretReplacementHintText; }); } - private removeSecretsFromSystems(systems: SystemEntity[]) { + private removeSecretsFromSystems(systems: SystemEntity[]): SystemEntity[] { systems.forEach((system) => { if (system.oauthConfig) { system.oauthConfig.clientSecret = defaultSecretReplacementHintText; @@ -428,16 +428,22 @@ export class DatabaseManagementUc { return systems; } + public async migrationCreate(): Promise { + await this.databaseManagementService.migrationCreate(); + } + public async migrationUp(from?: string, to?: string, only?: string): Promise { - return this.databaseManagementService.migrationUp(from, to, only); + await this.databaseManagementService.migrationUp(from, to, only); } public async migrationDown(from?: string, to?: string, only?: string): Promise { - return this.databaseManagementService.migrationDown(from, to, only); + await this.databaseManagementService.migrationDown(from, to, only); } public async migrationPending(): Promise { - return this.databaseManagementService.migrationPending(); + const result = await this.databaseManagementService.migrationPending(); + + return result; } public encryptPlainText(plainText: string): string { diff --git a/package.json b/package.json index d584b24d092..b6e6d9a3fe5 100644 --- a/package.json +++ b/package.json @@ -20,13 +20,6 @@ "node": "22", "npm": ">=9" }, - "mikro-orm": { - "useTsNode": true, - "configPaths": [ - "./apps/server/src/config/mikro-orm-cli.config.ts", - "./dist/server/config/mikro-orm-cli.config.js" - ] - }, "scripts": { "lint": "eslint . --ignore-path .gitignore", "test": "npm run nest:test && npm run feathers:test", @@ -53,6 +46,7 @@ "migration:down": "npm run nest:start:console -- database migration --down", "migration:pending": "npm run nest:start:console -- database migration --pending", "migration:persisted": "npm run nest:start:console -- database export --collection migrations --override", + "migration:create": "npm run nest:start:console -- database migration --create", "nest:prebuild": "rimraf dist", "nest:build": "nest build", "nest:build:all": "npm run nest:build", @@ -344,4 +338,4 @@ "tsconfig-paths": "^4.2.0", "typescript": "^5.7.3" } -} +} \ No newline at end of file From 0234dc430e4cd2f81bed22ca029d31aa7e64cccd Mon Sep 17 00:00:00 2001 From: SevenWaysDP Date: Mon, 10 Feb 2025 11:13:45 +0100 Subject: [PATCH 2/2] fix(mikro-orm): update migration paths for consistency in CLI configuration --- apps/server/src/config/mikro-orm-cli.config.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/apps/server/src/config/mikro-orm-cli.config.ts b/apps/server/src/config/mikro-orm-cli.config.ts index d1bb45a05f5..80792447d19 100644 --- a/apps/server/src/config/mikro-orm-cli.config.ts +++ b/apps/server/src/config/mikro-orm-cli.config.ts @@ -5,7 +5,19 @@ import { FileEntity } from '@modules/files/entity'; import { ALL_ENTITIES } from '@shared/domain/entity'; import path from 'path'; -const migrationsPath = path.resolve(__dirname, '..', 'migrations', 'mikro-orm'); +const migrationsDistPath = path.resolve(__dirname, '..', 'migrations', 'mikro-orm'); +const migrationsSourcePath = path.resolve( + __dirname, + '..', + '..', + '..', + '..', + 'apps', + 'server', + 'src', + 'migrations', + 'mikro-orm' +); export const mikroOrmCliConfig: MikroOrmModuleSyncOptions = { // TODO repeats server module definitions @@ -21,8 +33,8 @@ export const mikroOrmCliConfig: MikroOrmModuleSyncOptions = { */ migrations: { tableName: 'migrations', // name of database table with log of executed transactions - path: migrationsPath, // path to the folder with migrations - pathTs: migrationsPath.replace('/dist/apps/server/migrations/mikro-orm', '/apps/server/src/migrations/mikro-orm'), // path to the folder with TS migrations (if used, we should put path to compiled files in `path`) + path: migrationsDistPath, // path to the folder with migrations + pathTs: migrationsSourcePath, // path to the folder with TS migrations glob: '!(*.d).{js,ts}', // how to match migration files (all .js and .ts files, but not .d.ts) transactional: false, // wrap each migration in a transaction disableForeignKeys: true, // wrap statements with `set foreign_key_checks = 0` or equivalent