feat(all-services): Automate Migration Execution on Installation with Optional MySQL or PostgreSQL Migration Selection via Flag (#2209)

* feat(all-services): Automate Migration Execution on Installation with Optional MySQL or PostgreSQL Migration Selection via Flag

Automate Migration Execution on Installation with Optional MySQL or PostgreSQL Migration Selection via Flag

2207

* docs(all-services): update the readme of all services

update the readme of all services

2207

* fix(all-services): remove sonar

remove sonar

2207

* fix(all-services): lint fix

lint fix

2207

---------

Co-authored-by: yeshamavani <[email protected]>

Author: Tyagi-Sunny

services/audit-service/README.md

@@ -128,6 +128,10 @@ export class AuditDbDataSource
 
 The migrations required for this service are processed during the installation automatically if you set the `AUDIT_MIGRATION` or `SOURCELOOP_MIGRATION` env variable. The migrations use [`db-migrate`](https://www.npmjs.com/package/db-migrate) with [`db-migrate-pg`](https://www.npmjs.com/package/db-migrate-pg) driver for migrations, so you will have to install these packages to use auto-migration. Please note that if you are using some pre-existing migrations or databases, they may be affected. In such a scenario, it is advised that you copy the migration files in your project root, using the `AUDIT_MIGRATION_COPY` or `SOURCELOOP_MIGRATION_COPY` env variables. You can customize or cherry-pick the migrations in the copied files according to your specific requirements and then apply them to the DB.
 
+
+This migration script supports both MySQL and PostgreSQL databases, controlled by environment variables. By setting MYSQL_MIGRATION to 'true', the script runs migrations using MySQL configuration files; otherwise, it defaults to PostgreSQL. .
+
+
 Additionally, there is now an option to choose between SQL migration or PostgreSQL migration.
 NOTE : for [`@sourceloop/cli`](https://www.npmjs.com/package/@sourceloop/cli?activeTab=readme) users, this choice can be specified during the scaffolding process by selecting the "type of datasource" option.
 

services/audit-service/migration.js

@@ -2,13 +2,17 @@ const dotenvExt = require('dotenv-extended');
 const fs = require('fs');
 const path = require('path');
 let isLocal = false;
+
+// Load environment variables
 dotenvExt.load({
   path: path.join(process.env.INIT_CWD ?? '.', '.env'),
   defaults: path.join(process.env.INIT_CWD ?? '.', '.env.defaults'),
   errorOnMissing: false,
   includeProcessEnv: true,
 });
+
 const type = 'AUDIT';
+const isMysqlMigration = process.env.MYSQL_MIGRATION === 'true'; // Check for MySQL migration flag
 
 try {
   if (fs.existsSync('.infolder')) {
@@ -17,14 +21,14 @@ try {
 } catch (err) {
   console.info('\n');
 }
+
 if (isLocal) {
   console.info(`Skipping migrations`);
 } else if (
   !(process.env[`${type}_MIGRATION`] || process.env.SOURCELOOP_MIGRATION)
 ) {
   console.warn(
-    `${type}_MIGRATION or SOURCELOOP_MIGRATION variables not found in the environment,
-     skipping automigration.`,
+    `${type}_MIGRATION or SOURCELOOP_MIGRATION variables not found in the environment, skipping automigration.`,
   );
 } else {
   const DBMigrate = require('db-migrate');
@@ -34,50 +38,46 @@ if (isLocal) {
     errorOnMissing: true,
     includeProcessEnv: true,
   });
-  // Initialize db-migrate with correct migration directory and database config path
+
+  // Set migration directory and database config path based on MYSQL_MIGRATION flag
+  const migrationDir = isMysqlMigration ? 'mysql' : 'pg';
   const dbmigrate = DBMigrate.getInstance(true, {
-    config: path.join('migrations', 'pg', 'database.json'),
-    cwd: path.resolve(process.cwd(), 'migrations', 'pg'),
+    config: path.join('migrations', migrationDir, 'database.json'),
+    cwd: path.resolve(process.cwd(), 'migrations', migrationDir),
   });
+
   dbmigrate.up();
 }
 
+// Copy migration files if COPY flag is set
 if (
   process.env.SOURCELOOP_MIGRATION_COPY ||
   process.env[`${type}_MIGRATION_COPY`]
 ) {
+  const migrationDir = isMysqlMigration ? 'mysql' : 'pg';
   copyFolderRecursiveSync(
-    path.join('.', 'migrations', 'pg', 'migrations'),
+    path.join('.', 'migrations', migrationDir, 'migrations'),
     process.env.INIT_CWD,
   );
 }
 
+// Utility functions for copying files and folders
 function copyFileSync(source, target) {
   let targetFile = target;
-
-  // If target is a directory, a new file with the same name will be created
-  if (fs.existsSync(target)) {
-    if (fs.lstatSync(target).isDirectory()) {
-      targetFile = path.join(target, path.basename(source));
-    }
+  if (fs.existsSync(target) && fs.lstatSync(target).isDirectory()) {
+    targetFile = path.join(target, path.basename(source));
   }
-
   fs.writeFileSync(targetFile, fs.readFileSync(source));
 }
 
 function copyFolderRecursiveSync(source, target) {
-  let files = [];
-
-  // Check if folder needs to be created or integrated
-  let targetFolder = path.join(target, path.basename(source));
+  const targetFolder = path.join(target, path.basename(source));
   if (!fs.existsSync(targetFolder)) {
     fs.mkdirSync(targetFolder);
   }
-
-  // Copy
   if (fs.lstatSync(source).isDirectory()) {
-    files = fs.readdirSync(source);
-    files.forEach(function (file) {
+    const files = fs.readdirSync(source);
+    files.forEach(file => {
       const curSource = path.join(source, file);
       if (fs.lstatSync(curSource).isDirectory()) {
         copyFolderRecursiveSync(curSource, targetFolder);

services/authentication-service/README.md

@@ -485,6 +485,9 @@ export class AuthenticationDbDataSource
 
 The migrations required for this service are processed during the installation automatically if you set the `AUTH_MIGRATION` or `SOURCELOOP_MIGRATION` env variable. The migrations use [`db-migrate`](https://www.npmjs.com/package/db-migrate) with [`db-migrate-pg`](https://www.npmjs.com/package/db-migrate-pg) driver for migrations, so you will have to install these packages to use auto-migration. Please note that if you are using some pre-existing migrations or databases, they may be affected. In such a scenario, it is advised that you copy the migration files in your project root, using the `AUTH_MIGRATION_COPY` or `SOURCELOOP_MIGRATION_COPY` env variables. You can customize or cherry-pick the migrations in the copied files according to your specific requirements and then apply them to the DB.
 
+
+This migration script supports both MySQL and PostgreSQL databases, controlled by environment variables. By setting MYSQL_MIGRATION to 'true', the script runs migrations using MySQL configuration files; otherwise, it defaults to PostgreSQL. .
+
 Additionally, there is now an option to choose between SQL migration or PostgreSQL migration.
 NOTE : For [`@sourceloop/cli`](https://www.npmjs.com/package/@sourceloop/cli?activeTab=readme) users, this choice can be specified during the scaffolding process by selecting the "type of datasource" option.
 

services/authentication-service/migration.js

@@ -2,13 +2,17 @@ const dotenvExt = require('dotenv-extended');
 const fs = require('fs');
 const path = require('path');
 let isLocal = false;
+
+// Load environment variables
 dotenvExt.load({
   path: path.join(process.env.INIT_CWD ?? '.', '.env'),
   defaults: path.join(process.env.INIT_CWD ?? '.', '.env.defaults'),
   errorOnMissing: false,
   includeProcessEnv: true,
 });
+
 const type = 'AUTH';
+const isMysqlMigration = process.env.MYSQL_MIGRATION === 'true'; // Check for MySQL migration flag
 
 try {
   if (fs.existsSync('.infolder')) {
@@ -17,6 +21,7 @@ try {
 } catch (err) {
   console.info('\n');
 }
+
 if (isLocal) {
   console.info(`Skipping migrations`);
 } else if (
@@ -33,51 +38,46 @@ if (isLocal) {
     errorOnMissing: true,
     includeProcessEnv: true,
   });
-  // Initialize db-migrate with correct migration directory and database config path
+
+  // Set migration directory and database config path based on MYSQL_MIGRATION flag
+  const migrationDir = isMysqlMigration ? 'mysql' : 'pg';
   const dbmigrate = DBMigrate.getInstance(true, {
-    config: path.join('migrations', 'pg', 'database.json'),
-    cwd: path.resolve(process.cwd(), 'migrations', 'pg'),
+    config: path.join('migrations', migrationDir, 'database.json'),
+    cwd: path.resolve(process.cwd(), 'migrations', migrationDir),
   });
 
   dbmigrate.up();
 }
 
+// Copy migration files if COPY flag is set
 if (
   process.env.SOURCELOOP_MIGRATION_COPY ||
   process.env[`${type}_MIGRATION_COPY`]
 ) {
+  const migrationDir = isMysqlMigration ? 'mysql' : 'pg';
   copyFolderRecursiveSync(
-    path.join('.', 'migrations', 'pg', 'migrations'),
+    path.join('.', 'migrations', migrationDir, 'migrations'),
     process.env.INIT_CWD,
   );
 }
 
+// Utility functions for copying files and folders
 function copyFileSync(source, target) {
   let targetFile = target;
-
-  // If target is a directory, a new file with the same name will be created
-  if (fs.existsSync(target)) {
-    if (fs.lstatSync(target).isDirectory()) {
-      targetFile = path.join(target, path.basename(source));
-    }
+  if (fs.existsSync(target) && fs.lstatSync(target).isDirectory()) {
+    targetFile = path.join(target, path.basename(source));
   }
-
   fs.writeFileSync(targetFile, fs.readFileSync(source));
 }
 
 function copyFolderRecursiveSync(source, target) {
-  let files = [];
-
-  // Check if folder needs to be created or integrated
   const targetFolder = path.join(target, path.basename(source));
   if (!fs.existsSync(targetFolder)) {
     fs.mkdirSync(targetFolder);
   }
-
-  // Copy
   if (fs.lstatSync(source).isDirectory()) {
-    files = fs.readdirSync(source);
-    files.forEach(function (file) {
+    const files = fs.readdirSync(source);
+    files.forEach(file => {
       const curSource = path.join(source, file);
       if (fs.lstatSync(curSource).isDirectory()) {
         copyFolderRecursiveSync(curSource, targetFolder);

services/bpmn-service/README.md

@@ -195,6 +195,9 @@ A sample implementation of a `DataSource` using environment variables and Postgr
 
 The migrations required for this service are processed during the installation automatically if you set the `WORKFLOW_MIGRATION` or `SOURCELOOP_MIGRATION` env variable. The migrations use [`db-migrate`](https://www.npmjs.com/package/db-migrate) with [`db-migrate-pg`](https://www.npmjs.com/package/db-migrate-pg) driver for migrations, so you will have to install these packages to use auto-migration. Please note that if you are using some pre-existing migrations or databases, they may be affected. In such a scenario, it is advised that you copy the migration files in your project root, using the `WORKFLOW_MIGRATION_COPY` or `SOURCELOOP_MIGRATION_COPY` env variables. You can customize or cherry-pick the migrations in the copied files according to your specific requirements and then apply them to the DB.
 
+
+This migration script supports both MySQL and PostgreSQL databases, controlled by environment variables. By setting MYSQL_MIGRATION to 'true', the script runs migrations using MySQL configuration files; otherwise, it defaults to PostgreSQL. .
+
 This project includes no migrations to seed your BPMN engine. If you are using Camunda BPM Run, you can use either the `resources` folder to seed a model, or you can config it to use a custom DB where you can seed your data. The steps to config Platform Run are given [here](https://camunda.com/blog/2020/03/introducing-camunda-bpm-run/).
 
 Additionally, there is now an option to choose between SQL migration or PostgreSQL migration.

services/bpmn-service/migration.js

@@ -2,13 +2,17 @@ const dotenvExt = require('dotenv-extended');
 const fs = require('fs');
 const path = require('path');
 let isLocal = false;
+
+// Load environment variables
 dotenvExt.load({
   path: path.join(process.env.INIT_CWD ?? '.', '.env'),
   defaults: path.join(process.env.INIT_CWD ?? '.', '.env.defaults'),
   errorOnMissing: false,
   includeProcessEnv: true,
 });
+
 const type = 'WORKFLOW';
+const isMysqlMigration = process.env.MYSQL_MIGRATION === 'true'; // Check for MySQL migration flag
 
 try {
   if (fs.existsSync('.infolder')) {
@@ -17,6 +21,7 @@ try {
 } catch (err) {
   console.info('\n');
 }
+
 if (isLocal) {
   console.info(`Skipping migrations`);
 } else if (
@@ -33,50 +38,46 @@ if (isLocal) {
     errorOnMissing: true,
     includeProcessEnv: true,
   });
-  // Initialize db-migrate with correct migration directory and database config path
+
+  // Set migration directory and database config path based on MYSQL_MIGRATION flag
+  const migrationDir = isMysqlMigration ? 'mysql' : 'pg';
   const dbmigrate = DBMigrate.getInstance(true, {
-    config: path.join('migrations', 'pg', 'database.json'),
-    cwd: path.resolve(process.cwd(), 'migrations', 'pg'),
+    config: path.join('migrations', migrationDir, 'database.json'),
+    cwd: path.resolve(process.cwd(), 'migrations', migrationDir),
   });
+
   dbmigrate.up();
 }
 
+// Copy migration files if COPY flag is set
 if (
   process.env.SOURCELOOP_MIGRATION_COPY ||
   process.env[`${type}_MIGRATION_COPY`]
 ) {
+  const migrationDir = isMysqlMigration ? 'mysql' : 'pg';
   copyFolderRecursiveSync(
-    path.join('.', 'migrations', 'pg', 'migrations'),
+    path.join('.', 'migrations', migrationDir, 'migrations'),
     process.env.INIT_CWD,
   );
 }
 
+// Utility functions for copying files and folders
 function copyFileSync(source, target) {
   let targetFile = target;
-
-  // If target is a directory, a new file with the same name will be created
-  if (fs.existsSync(target)) {
-    if (fs.lstatSync(target).isDirectory()) {
-      targetFile = path.join(target, path.basename(source));
-    }
+  if (fs.existsSync(target) && fs.lstatSync(target).isDirectory()) {
+    targetFile = path.join(target, path.basename(source));
   }
-
   fs.writeFileSync(targetFile, fs.readFileSync(source));
 }
 
 function copyFolderRecursiveSync(source, target) {
-  let files = [];
-
-  // Check if folder needs to be created or integrated
   const targetFolder = path.join(target, path.basename(source));
   if (!fs.existsSync(targetFolder)) {
     fs.mkdirSync(targetFolder);
   }
-
-  // Copy
   if (fs.lstatSync(source).isDirectory()) {
-    files = fs.readdirSync(source);
-    files.forEach(function (file) {
+    const files = fs.readdirSync(source);
+    files.forEach(file => {
       const curSource = path.join(source, file);
       if (fs.lstatSync(curSource).isDirectory()) {
         copyFolderRecursiveSync(curSource, targetFolder);

services/chat-service/README.md

@@ -147,6 +147,9 @@ export class ChatDataSource
 
 The migrations required for this service are processed during the installation automatically if you set the `CHAT_MIGRATION` or `SOURCELOOP_MIGRATION` env variable. The migrations use [`db-migrate`](https://www.npmjs.com/package/db-migrate) with [`db-migrate-pg`](https://www.npmjs.com/package/db-migrate-pg) driver for migrations, so you will have to install these packages to use auto-migration. Please note that if you are using some pre-existing migrations or databasea, they may be affected. In such a scenario, it is advised that you copy the migration files in your project root, using the `CHAT_MIGRATION_COPY` or `SOURCELOOP_MIGRATION_COPY` env variables. You can customize or cherry-pick the migrations in the copied files according to your specific requirements and then apply them to the DB.
 
+
+This migration script supports both MySQL and PostgreSQL databases, controlled by environment variables. By setting MYSQL_MIGRATION to 'true', the script runs migrations using MySQL configuration files; otherwise, it defaults to PostgreSQL. .
+
 Additionally, there is now an option to choose between SQL migration or PostgreSQL migration.
 NOTE: For [`@sourceloop/cli`](https://www.npmjs.com/package/@sourceloop/cli?activeTab=readme) users, this choice can be specified during the scaffolding process by selecting the "type of datasource" option.